commit 780405973dd11a06ee19c15b9c482ae436886b6e parent bb7dd490c6ed19f5eb19c2f72d06dc8d2058a541 Author: Kevin Corvisier <git@kevincorvisier.fr> Date: Sat, 19 Oct 2024 18:02:19 +0900 Use the Spring Framework to load configuration files and inject dependencies Diffstat:
27 files changed, 744 insertions(+), 685 deletions(-)
diff --git a/lombok.config b/lombok.config @@ -0,0 +1 @@ +lombok.copyableAnnotations += org.springframework.beans.factory.annotation.Value +\ No newline at end of file diff --git a/pom.xml b/pom.xml @@ -13,12 +13,39 @@ <maven.compiler.source>17</maven.compiler.source> <htmlunit-driver.version>3.58.0</htmlunit-driver.version> + <junit.version>5.11.2</junit.version> <logback.version>1.2.10</logback.version> <lombok.version>1.18.22</lombok.version> <sqlite-jdbc.version>3.45.0.0</sqlite-jdbc.version> + <spring.version>6.1.14</spring.version> </properties> + <dependencyManagement> + <dependencies> + <dependency> + <groupId>org.junit</groupId> + <artifactId>junit-bom</artifactId> + <version>${junit.version}</version> + <type>pom</type> + <scope>import</scope> + </dependency> + + <dependency> + <groupId>org.springframework</groupId> + <artifactId>spring-framework-bom</artifactId> + <version>${spring.version}</version> + <type>pom</type> + <scope>import</scope> + </dependency> + </dependencies> + </dependencyManagement> + <dependencies> + <dependency> + <groupId>org.springframework</groupId> + <artifactId>spring-context</artifactId> + </dependency> + <!--HtmlUnit Driver --> <dependency> <groupId>org.seleniumhq.selenium</groupId> @@ -60,13 +87,11 @@ <dependency> <groupId>org.junit.jupiter</groupId> <artifactId>junit-jupiter-api</artifactId> - <version>5.10.2</version> <scope>test</scope> </dependency> <dependency> <groupId>org.junit.jupiter</groupId> <artifactId>junit-jupiter-engine</artifactId> - <version>5.10.2</version> <scope>test</scope> </dependency> </dependencies> diff --git a/src/main/java/fr/kevincorvisier/mtg/dd/Cache.java b/src/main/java/fr/kevincorvisier/mtg/dd/Cache.java @@ -12,15 +12,17 @@ import java.sql.ResultSet; import java.sql.SQLException; import java.time.LocalDate; import java.util.Optional; -import java.util.Properties; import org.apache.commons.io.output.StringBuilderWriter; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Service; import fr.kevincorvisier.mtg.dd.model.DeckItem; import fr.kevincorvisier.mtg.dd.model.DeckItemFactory; import fr.kevincorvisier.mtg.dd.mtgo.MagicOnlineFileReader; import fr.kevincorvisier.mtg.dd.mtgo.MagicOnlineFileWriter; +@Service public class Cache { private static final String QUERY_INITIALIZE = "CREATE TABLE IF NOT EXISTS `decks` (`url` TEXT PRIMARY KEY,`player` TEXT,`archetype` TEXT,`date` TEXT,`content` TEXT);"; @@ -28,9 +30,8 @@ public class Cache private final Connection connection; private final DeckItemFactory deckItemFactory; - public Cache(final DeckItemFactory deckItemFactory, final Properties props) throws SQLException + public Cache(final DeckItemFactory deckItemFactory, @Value("${cache.database}") final String file) throws SQLException { - final String file = props.getProperty("cache.database"); if (file == null) throw new RuntimeException("Property cache.database must be present and have a value"); diff --git a/src/main/java/fr/kevincorvisier/mtg/dd/Crawler.java b/src/main/java/fr/kevincorvisier/mtg/dd/Crawler.java @@ -9,12 +9,14 @@ import org.openqa.selenium.By; import org.openqa.selenium.WebDriver; import org.openqa.selenium.WebElement; import org.openqa.selenium.htmlunit.HtmlUnitDriver; +import org.springframework.stereotype.Service; import com.google.common.util.concurrent.RateLimiter; import lombok.extern.slf4j.Slf4j; @Slf4j +@Service public class Crawler { private static final double CRAWL_DELAY_SECONDS = 30d; diff --git a/src/main/java/fr/kevincorvisier/mtg/dd/DecklistConsumer.java b/src/main/java/fr/kevincorvisier/mtg/dd/DecklistConsumer.java @@ -1,16 +0,0 @@ -package fr.kevincorvisier.mtg.dd; - -import java.io.IOException; - -import fr.kevincorvisier.mtg.dd.model.DeckItem; - -public interface DecklistConsumer -{ - int capacity(); - - int capacity(final DeckItem deck); - - void consume(final DeckItem deck) throws IOException; - - void saveToFolder() throws IOException; -} diff --git a/src/main/java/fr/kevincorvisier/mtg/dd/DecklistDownloader.java b/src/main/java/fr/kevincorvisier/mtg/dd/DecklistDownloader.java @@ -1,11 +0,0 @@ -package fr.kevincorvisier.mtg.dd; - -import java.io.IOException; -import java.net.URL; - -public interface DecklistDownloader -{ - boolean accept(final URL url); - - void download(final URL url, final DecklistConsumer consumer) throws IOException; -} diff --git a/src/main/java/fr/kevincorvisier/mtg/dd/DefaultDecklistConsumer.java b/src/main/java/fr/kevincorvisier/mtg/dd/DefaultDecklistConsumer.java @@ -1,160 +0,0 @@ -package fr.kevincorvisier.mtg.dd; - -import java.io.BufferedReader; -import java.io.BufferedWriter; -import java.io.File; -import java.io.FileWriter; -import java.io.IOException; -import java.io.InputStreamReader; -import java.nio.file.Files; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Map; -import java.util.Set; -import java.util.function.Function; -import java.util.stream.Collectors; - -import org.apache.commons.lang3.StringUtils; - -import fr.kevincorvisier.mtg.dd.model.DeckItem; -import fr.kevincorvisier.mtg.dd.model.DeckItemComparator; -import fr.kevincorvisier.mtg.dd.mtgo.MagicOnlineFileReader; -import fr.kevincorvisier.mtg.dd.mtgo.MagicOnlineFileWriter; -import fr.kevincorvisier.mtg.dd.validation.DeckValidator; -import lombok.RequiredArgsConstructor; -import lombok.extern.slf4j.Slf4j; - -@Slf4j -@RequiredArgsConstructor -public class DefaultDecklistConsumer implements DecklistConsumer -{ - private final Map<String, Map<String, DeckItem>> downloadedByDeckNameByPlayerName = new HashMap<>(); - private final Set<Integer> downloadedContent = new HashSet<>(); - private final Map<String, Integer> countByArchetype = new HashMap<>(); - - private final Cache cache; - private final File folder; - private final int limit; - private final int archetypeLimit; - private final DeckValidator validator; - private final Crawler crawler; - private final String[] ignorePlayers; - private final boolean onlyAiPlayableCards; - - @Override - public int capacity() - { - return limit - downloadedContent.size(); - } - - @Override - public int capacity(final DeckItem deck) - { - final int globalCapacity = capacity(); - final int archetypeCapacity = archetypeLimit - countByArchetype.getOrDefault(deck.getArchetype(), 0); - - return Math.min(archetypeCapacity, globalCapacity); - } - - private boolean accept(final DeckItem deck) - { - if (capacity(deck) <= 0) - return false; - - final String playerName = deck.getPlayer().toLowerCase(); - final String deckName = deck.getArchetype().toLowerCase(); - - if (StringUtils.containsAnyIgnoreCase(playerName, ignorePlayers)) - return false; // Ignore this player - - final DeckItem previouslyDownloadedVersion = downloadedByDeckNameByPlayerName.getOrDefault(playerName, Collections.emptyMap()).get(deckName); - - if (previouslyDownloadedVersion != null && previouslyDownloadedVersion.getDate().isAfter(deck.getDate())) - { - log.info("consume: more recent version of {} {} already downloaded, ignoring", deck.getPlayer(), deck.getArchetype()); - return false; - } - - return true; - } - - @Override - public void consume(final DeckItem deck) throws IOException - { - try - { - if (!accept(deck)) - return; - - final DeckItem cached = cache.findByUrl(deck.getUrl()).orElse(null); - if (cached != null) - deck.setMain(cached.getMain()); - else - { - try (BufferedReader reader = new BufferedReader(new InputStreamReader(crawler.openStream(deck.getUrl())))) - { - deck.setMain(MagicOnlineFileReader.read(reader)); - } - cache.save(deck); - } - - if (downloadedContent.contains(deck.getMain().hashCode())) - { - log.warn("consume: ignore deck: same decklist already exists: {}", deck); - return; - } - - if (!validator.validate(deck, onlyAiPlayableCards)) - return; - - downloadedByDeckNameByPlayerName.computeIfAbsent(deck.getPlayer().toLowerCase(), k -> new HashMap<>()) // - .put(deck.getArchetype().toLowerCase(), deck); - - downloadedContent.add(deck.getMain().hashCode()); - countByArchetype.merge(deck.getArchetype(), 1, (a, b) -> a + b); - } - catch (final Exception e) - { - log.error("Error while downloading deck, skipping {}", deck, e); - } - } - - @Override - public void saveToFolder() throws IOException - { - if (!folder.exists()) - folder.mkdirs(); - - final Map<String, DeckItem> deckByFileName = downloadedByDeckNameByPlayerName.values().stream() // - .flatMap(v -> v.values().stream()) // - .collect(Collectors.toMap(DeckItem::getFilename, Function.identity())); - - for (final File file : folder.listFiles()) - { - if (deckByFileName.containsKey(file.getName())) - { - deckByFileName.remove(file.getName()); - continue; - } - - Files.delete(file.toPath()); - log.info("Deleted {}", file); - } - - final Set<DeckItem> toSave = deckByFileName.values().stream() // - .sorted(DeckItemComparator.INSTANCE) // - .limit(limit) // - .collect(Collectors.toUnmodifiableSet()); - - for (final DeckItem deck : toSave) - { - final File file = new File(folder, deck.getFilename()); - try (BufferedWriter writer = new BufferedWriter(new FileWriter(file))) - { - MagicOnlineFileWriter.write(writer, deck); - } - log.info("Saved {}", file); - } - } -} diff --git a/src/main/java/fr/kevincorvisier/mtg/dd/Main.java b/src/main/java/fr/kevincorvisier/mtg/dd/Main.java @@ -2,112 +2,76 @@ package fr.kevincorvisier.mtg.dd; import java.io.File; import java.io.IOException; +import java.net.URISyntaxException; import java.net.URL; -import java.sql.SQLException; -import java.util.ArrayList; -import java.util.Arrays; import java.util.Collection; -import java.util.List; -import java.util.Map; -import java.util.Properties; -import java.util.stream.Collectors; -import org.apache.commons.lang3.StringUtils; +import org.springframework.beans.BeansException; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.AnnotationConfigApplicationContext; +import org.springframework.core.io.support.ResourcePropertySource; +import org.springframework.stereotype.Service; -import fr.kevincorvisier.mtg.dd.hareruya.HareruyaDecklistDownloader; -import fr.kevincorvisier.mtg.dd.model.DeckItemFactory; -import fr.kevincorvisier.mtg.dd.tcdecks.TcdecksDecklistDownloader; -import fr.kevincorvisier.mtg.dd.validation.DeckValidator; -import fr.kevincorvisier.mtg.dd.validation.DeckValidatorFactory; +import fr.kevincorvisier.mtg.dd.consumers.DecklistConsumer; +import fr.kevincorvisier.mtg.dd.downloaders.DecklistDownloader; +import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; @Slf4j +@Service +@RequiredArgsConstructor public class Main { - public static void main(final String[] args) throws IOException, SQLException - { - final Properties props = new Properties(); - props.load(Main.class.getClassLoader().getResourceAsStream("application.properties")); - - final List<String> formats = Arrays.stream(props.getProperty("formats", "").split(",")) // - .map(String::trim).filter(StringUtils::isNotEmpty) // - .distinct().toList(); - - final String[] ignorePlayers = Arrays.stream(props.getProperty("ignore.players", "").split(",")) // - .map(String::trim).filter(StringUtils::isNotEmpty) // - .toArray(String[]::new); - - final Map<String, DeckValidator> validatorByFormat = DeckValidatorFactory.create(formats, props); - - final Map<String, String> replaceArchetypes = Arrays.stream(props.getProperty("replace-archetypes", "").split(",")) // - .map(String::trim).filter(StringUtils::isNotEmpty) // - .collect(Collectors.toMap(s -> s.split("\\|")[0].trim(), s -> s.split("\\|")[1].trim())); + private final DecklistConsumer consumer; + private final Collection<DecklistDownloader> downloaders; - final DeckItemFactory deckItemFactory = new DeckItemFactory(replaceArchetypes); - final Cache cache = new Cache(deckItemFactory, props); - final Crawler crawler = new Crawler(); + @Value("#{'${sources}'.split('\\|')}") + private final Collection<URL> sources; - final Collection<DecklistDownloader> downloaders = List.of(new HareruyaDecklistDownloader(crawler, deckItemFactory), - new TcdecksDecklistDownloader(crawler, deckItemFactory, props)); - - for (final String format : formats) + public void run() throws IOException + { + try { - final Collection<URL> urls = new ArrayList<>(); - int i = 0; - - while (true) + for (final URL url : sources) { - final String url = props.getProperty("formats." + format + ".sources." + i++); - if (url == null) - break; - urls.add(new URL(url)); - } - - final String outputFolder = props.getProperty("formats." + format + ".output-dir"); - if (outputFolder == null) - { - log.warn("main: no output folder for format, skipping: {}", format); - continue; - } - - final int limit = Integer.parseInt(props.getProperty("formats." + format + ".limit", "0")); - if (limit <= 0) - { - log.warn("main: limit invalid, skipping: {}", limit); - continue; - } - - final boolean onlyAiPlayableCards = Boolean.parseBoolean(props.getProperty("formats." + format + ".only-ai-playable-cards", "false")); + final DecklistDownloader downloader = downloaders.stream().filter(d -> d.accept(url)).findAny().orElse(null); + if (downloader == null) + { + log.warn("No downloader for URL: {}", url); + continue; + } - final int archetypeLimit = Integer.parseInt(props.getProperty("formats." + format + ".archetype-limit", "0")); - if (limit <= 0) - { - log.warn("main: archetype limit invalid, skipping: {}", limit); - continue; + downloader.download(url); } + } + finally + { + consumer.saveToFolder(); + } + } - final DeckValidator validator = validatorByFormat.get(format); - final DecklistConsumer consumer = new DefaultDecklistConsumer(cache, new File(outputFolder), limit, archetypeLimit, validator, crawler, - ignorePlayers, onlyAiPlayableCards); + public static void main(final String[] args) throws IOException, BeansException, IllegalStateException, URISyntaxException + { + for (final File file : getConfigurations()) + { + final File realFile = file.toPath().toRealPath().toFile(); - try + try (final AnnotationConfigApplicationContext context = new AnnotationConfigApplicationContext()) { - for (final URL url : urls) - { - final DecklistDownloader downloader = downloaders.stream().filter(d -> d.accept(url)).findAny().orElse(null); - if (downloader == null) - { - log.warn("No downloader for URL: {}", url); - continue; - } + context.getEnvironment().getPropertySources().addLast(new ResourcePropertySource("application.properties")); + context.getEnvironment().getPropertySources().addLast(new ResourcePropertySource(realFile.toURI().toString())); + context.scan("fr.kevincorvisier.mtg.dd"); + context.refresh(); - downloader.download(url, consumer); - } - } - finally - { - consumer.saveToFolder(); + context.getBean(Main.class).run(); } } } + + private static File[] getConfigurations() throws URISyntaxException + { + final File configEnabled = new File(Main.class.getClassLoader().getResource("config-enabled").toURI()); + + return configEnabled.listFiles(pathname -> pathname.isFile() && pathname.getName().endsWith(".properties")); + } } diff --git a/src/main/java/fr/kevincorvisier/mtg/dd/consumers/DecklistConsumer.java b/src/main/java/fr/kevincorvisier/mtg/dd/consumers/DecklistConsumer.java @@ -0,0 +1,16 @@ +package fr.kevincorvisier.mtg.dd.consumers; + +import java.io.IOException; + +import fr.kevincorvisier.mtg.dd.model.DeckItem; + +public interface DecklistConsumer +{ + int capacity(); + + int capacity(final DeckItem deck); + + void consume(final DeckItem deck); + + void saveToFolder() throws IOException; +} diff --git a/src/main/java/fr/kevincorvisier/mtg/dd/consumers/DefaultDecklistConsumer.java b/src/main/java/fr/kevincorvisier/mtg/dd/consumers/DefaultDecklistConsumer.java @@ -0,0 +1,171 @@ +package fr.kevincorvisier.mtg.dd.consumers; + +import java.io.BufferedReader; +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.io.InputStreamReader; +import java.nio.file.Files; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import java.util.function.Function; +import java.util.stream.Collectors; + +import org.apache.commons.lang3.StringUtils; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Service; + +import fr.kevincorvisier.mtg.dd.Cache; +import fr.kevincorvisier.mtg.dd.Crawler; +import fr.kevincorvisier.mtg.dd.model.DeckItem; +import fr.kevincorvisier.mtg.dd.model.DeckItemComparator; +import fr.kevincorvisier.mtg.dd.mtgo.MagicOnlineFileReader; +import fr.kevincorvisier.mtg.dd.mtgo.MagicOnlineFileWriter; +import fr.kevincorvisier.mtg.dd.validation.DeckValidator; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +@Service +@RequiredArgsConstructor +public class DefaultDecklistConsumer implements DecklistConsumer +{ + private final Map<String, Map<String, DeckItem>> downloadedByDeckNameByPlayerName = new HashMap<>(); + private final Set<Integer> downloadedContent = new HashSet<>(); + private final Map<String, Integer> countByArchetype = new HashMap<>(); + + private final Cache cache; + private final DeckValidator validator; + private final Crawler crawler; + + @Value("${only-ai-playable-cards}") + private final boolean onlyAiPlayableCards; + @Value("${output-dir}") + private final File folder; + @Value("${ignore.players}") + private final String[] ignorePlayers; + @Value("${limit}") + private final int limit; + @Value("${archetype-limit}") + private final int archetypeLimit; + + @Override + public int capacity() + { + return limit - downloadedContent.size(); + } + + @Override + public int capacity(final DeckItem deck) + { + final int globalCapacity = capacity(); + final int archetypeCapacity = archetypeLimit - countByArchetype.getOrDefault(deck.getArchetype(), 0); + + return Math.min(archetypeCapacity, globalCapacity); + } + + private boolean accept(final DeckItem deck) + { + if (capacity(deck) <= 0) + return false; + + final String playerName = deck.getPlayer().toLowerCase(); + final String deckName = deck.getArchetype().toLowerCase(); + + if (StringUtils.containsAnyIgnoreCase(playerName, ignorePlayers)) + return false; // Ignore this player + + final DeckItem previouslyDownloadedVersion = downloadedByDeckNameByPlayerName.getOrDefault(playerName, Collections.emptyMap()).get(deckName); + + if (previouslyDownloadedVersion != null && previouslyDownloadedVersion.getDate().isAfter(deck.getDate())) + { + log.info("consume: more recent version of {} {} already downloaded, ignoring", deck.getPlayer(), deck.getArchetype()); + return false; + } + + return true; + } + + @Override + public void consume(final DeckItem deck) + { + try + { + if (!accept(deck)) + return; + + final DeckItem cached = cache.findByUrl(deck.getUrl()).orElse(null); + if (cached != null) + deck.setMain(cached.getMain()); + else + { + try (BufferedReader reader = new BufferedReader(new InputStreamReader(crawler.openStream(deck.getUrl())))) + { + deck.setMain(MagicOnlineFileReader.read(reader)); + } + cache.save(deck); + } + + if (downloadedContent.contains(deck.getMain().hashCode())) + { + log.warn("consume: ignore deck: same decklist already exists: {}", deck); + return; + } + + if (!validator.validate(deck, onlyAiPlayableCards)) + return; + + downloadedByDeckNameByPlayerName.computeIfAbsent(deck.getPlayer().toLowerCase(), k -> new HashMap<>()) // + .put(deck.getArchetype().toLowerCase(), deck); + + downloadedContent.add(deck.getMain().hashCode()); + countByArchetype.merge(deck.getArchetype(), 1, (a, b) -> a + b); + } + catch (final Exception e) + { + log.error("Error while downloading deck, skipping {}", deck, e); + } + } + + @Override + public void saveToFolder() throws IOException + { + if (!folder.exists()) + folder.mkdirs(); + + final Map<String, DeckItem> deckByFileName = downloadedByDeckNameByPlayerName.values().stream() // + .flatMap(v -> v.values().stream()) // + .collect(Collectors.toMap(DeckItem::getFilename, Function.identity())); + + for (final File file : folder.listFiles()) + { + if (deckByFileName.containsKey(file.getName())) + { + deckByFileName.remove(file.getName()); + continue; + } + + Files.delete(file.toPath()); + log.info("Deleted {}", file); + } + + final Set<DeckItem> toSave = deckByFileName.values().stream() // + .sorted(DeckItemComparator.INSTANCE) // + .limit(limit) // + .collect(Collectors.toUnmodifiableSet()); + + for (final DeckItem deck : toSave) + { + final File file = new File(folder, deck.getFilename()); + try (BufferedWriter writer = new BufferedWriter(new FileWriter(file))) + { + MagicOnlineFileWriter.write(writer, deck); + } + log.info("Saved {}", file); + } + } +} diff --git a/src/main/java/fr/kevincorvisier/mtg/dd/downloaders/DecklistDownloader.java b/src/main/java/fr/kevincorvisier/mtg/dd/downloaders/DecklistDownloader.java @@ -0,0 +1,11 @@ +package fr.kevincorvisier.mtg.dd.downloaders; + +import java.net.MalformedURLException; +import java.net.URL; + +public interface DecklistDownloader +{ + boolean accept(final URL url); + + void download(final URL url) throws MalformedURLException; +} diff --git a/src/main/java/fr/kevincorvisier/mtg/dd/downloaders/HareruyaDecklistDownloader.java b/src/main/java/fr/kevincorvisier/mtg/dd/downloaders/HareruyaDecklistDownloader.java @@ -0,0 +1,74 @@ +package fr.kevincorvisier.mtg.dd.downloaders; + +import java.io.IOException; +import java.net.URL; +import java.time.LocalDate; + +import org.openqa.selenium.By; +import org.openqa.selenium.WebElement; +import org.springframework.stereotype.Service; + +import fr.kevincorvisier.mtg.dd.Crawler; +import fr.kevincorvisier.mtg.dd.consumers.DecklistConsumer; +import fr.kevincorvisier.mtg.dd.model.DeckItem; +import fr.kevincorvisier.mtg.dd.model.DeckItemFactory; +import lombok.RequiredArgsConstructor; + +@Service +@RequiredArgsConstructor +public class HareruyaDecklistDownloader implements DecklistDownloader +{ + private final Crawler crawler; + private final DeckItemFactory deckItemFactory; + private final DecklistConsumer consumer; + + @Override + public boolean accept(final URL url) + { + return url.getHost().equals("www.hareruyamtg.com"); + } + + @Override + public void download(final URL url) + { + crawler.navigateTo(url); + + for (final WebElement element : crawler.findElements(By.className("deckSearch-searchResult__itemWrapper"))) + { + try + { + final DeckItem deck = processDeckElement(element); + consumer.consume(deck); + } + catch (final IOException e) + { + e.printStackTrace(); + } + } + } + + private DeckItem processDeckElement(final WebElement element) throws IOException + { + final String playerName = element.findElement(By.className("deckSearch-searchResult__item__playerName__link")).getText(); + final String deckName = element.findElement(By.xpath(".//li[@class='deckSearch-searchResult__item__deckName']/span[2]")).getText(); + final String date = parseDate(element, By.className("deckSearch-searchResult__item__tournamentDate")); + + // Retrieve the id from the URL, then build the download URL from it + // Format: https://www.hareruyamtg.com/en/deck/361764/show/ + final String href = element.getAttribute("href"); + final int beginIndex = "https://www.hareruyamtg.com/en/deck/".length(); + final int endIndex = href.length() - "/show/".length(); + final long id = Long.parseLong(href.substring(beginIndex, endIndex)); + final URL url = new URL("https://www.hareruyamtg.com/en/deck/download/" + id); + + return deckItemFactory.create(url, playerName, deckName, LocalDate.parse(date)); + } + + private String parseDate(final WebElement element, final By by) + { + String text = element.findElement(by).getText(); + if (text.startsWith("Date：")) + text = text.substring("Date：".length()); + return text.replace('/', '-').replace(" ", ""); + } +} diff --git a/src/main/java/fr/kevincorvisier/mtg/dd/downloaders/TcdecksDecklistDownloader.java b/src/main/java/fr/kevincorvisier/mtg/dd/downloaders/TcdecksDecklistDownloader.java @@ -0,0 +1,210 @@ +package fr.kevincorvisier.mtg.dd.downloaders; + +import java.net.MalformedURLException; +import java.net.URL; +import java.time.LocalDate; +import java.time.format.DateTimeFormatter; +import java.util.ArrayList; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.openqa.selenium.By; +import org.openqa.selenium.NoSuchElementException; +import org.openqa.selenium.WebElement; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Service; + +import fr.kevincorvisier.mtg.dd.Crawler; +import fr.kevincorvisier.mtg.dd.consumers.DecklistConsumer; +import fr.kevincorvisier.mtg.dd.model.DeckItemFactory; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +@Service +@RequiredArgsConstructor +public class TcdecksDecklistDownloader implements DecklistDownloader +{ + private static final Pattern PATTERN_URL_FORMAT = Pattern.compile("https://www\\.tcdecks\\.net/format\\.php\\?format=(?:[A-Za-z])+"); + private static final Pattern PATTERN_URL_ARCHETYPE = Pattern + .compile("https://www\\.tcdecks\\.net/archetype\\.php\\?archetype=(?<archetype>[A-Za-z]+)&format=[A-Za-z]+"); + private static final Pattern PATTERN_URL_DECK = Pattern.compile("https://www.tcdecks.net/deck\\.php\\?id=(?<id>\\d+)&iddeck=(?<iddeck>\\d+)"); + + private static final Pattern PATTERN_POSITION = Pattern.compile("\\d+ of (?<players>\\d+)"); + + private static final Pattern pattern = Pattern.compile("Date: (?<dayOfMonth>\\d{2})\\/(?<month>\\d{2})\\/(?<year>\\d{4})"); + private static final Pattern PATTERN_PLAYERS = Pattern.compile("(?<players>\\d+) Players"); + + private static final String FORMAT_DOWNLOAD_URL = "/download.php?ext=txt&id=%1$s&iddeck=%2$s"; + + private final Crawler crawler; + private final DeckItemFactory deckItemFactory; + private final DecklistConsumer consumer; + + @Value("${tcdecks.tournament.players.min}") + private final int tournamentMinPlayers; + + @Override + public boolean accept(final URL url) + { + return url.getHost().equals("www.tcdecks.net"); + } + + @Override + public void download(final URL url) throws MalformedURLException + { + final String urlStr = url.toString(); + + if (PATTERN_URL_FORMAT.matcher(urlStr).matches()) + { + downloadFormat(url); + return; + } + + final Matcher matcher = PATTERN_URL_ARCHETYPE.matcher(urlStr); + if (matcher.matches()) + { + final String archetype = matcher.group("archetype"); + downloadArchetype(url, archetype); + } + } + + private void downloadFormat(final URL url) throws MalformedURLException + { + URL next = url; + + do + { + crawler.navigateTo(next); + + final List<URL> tournaments = new ArrayList<>(); + + for (final WebElement tr : crawler.findElements(By.xpath("//table[@class='tourney_list']/tbody/tr"))) + { + if (hasElement(tr, By.xpath("th"))) + continue; // Header row + + final WebElement tournamentName = tr.findElement(By.xpath("td[@data-th='Tournament Name']/a")); + final String players = tr.findElement(By.xpath("td[@data-th='Players']/a")).getText(); + + if ("Unknown".equals(players)) + continue; // Skip tournaments with no player count + + final Matcher matcher = PATTERN_PLAYERS.matcher(players); + if (!matcher.matches()) + { + log.error("Cannot parse players column: {}", players); + return; + } + + final int nbPlayers = Integer.parseInt(matcher.group("players")); + if (nbPlayers < tournamentMinPlayers) + { + log.info("Skip {}: less than {} players ({} players)", tournamentName.getText(), tournamentMinPlayers, nbPlayers); + continue; // Skip small tournaments + } + + tournaments.add(new URL(tournamentName.getAttribute("href"))); + } + + next = new URL(crawler.findElement(By.xpath("//a[text()='Next']")).getAttribute("href")); + + for (final URL tournament : tournaments) + downloadTournament(tournament); + } while (consumer.capacity() > 0); + } + + private void downloadArchetype(final URL url, final String archetype) throws MalformedURLException + { + URL next = url; + + do + { + crawler.navigateTo(next); + + next = new URL(crawler.findElement(By.xpath("//a[text()='Next']")).getAttribute("href")); + + for (final WebElement tr : crawler.findElements(By.xpath("//table[@class='tourney_list']/tbody/tr"))) + { + if (hasElement(tr, By.xpath("th"))) + continue; // Header row + + final WebElement deckName = tr.findElement(By.xpath("td[@data-th='Deck Name']/a")); + final WebElement player = tr.findElement(By.xpath("td[@data-th='Player']/a")); + final WebElement date = tr.findElement(By.xpath("td[@data-th='Date']/a")); + final WebElement position = tr.findElement(By.xpath("td[@data-th='Position']/a")); + final URL downloadUrl = toDownloadUrl(deckName.getAttribute("href")); + + final Matcher matcher = PATTERN_POSITION.matcher(position.getText()); + if (matcher.matches()) + { + final int nbPlayers = Integer.parseInt(matcher.group("players")); + if (nbPlayers < tournamentMinPlayers) + { + log.info("Skip {}: less than {} players ({} players)", deckName.getText(), tournamentMinPlayers, nbPlayers); + continue; // Skip small tournaments + } + } + + consumer.consume(deckItemFactory.create(downloadUrl, player.getText(), archetype, + LocalDate.parse(date.getText(), DateTimeFormatter.ofPattern("dd/MM/yyyy")))); + } + } while (consumer.capacity() > 0); + } + + private void downloadTournament(final URL url) throws MalformedURLException + { + crawler.navigateTo(url); + + final String description = crawler.findElement(By.xpath("//article[@class='span8']/fieldset/legend/h5")).getText(); + + log.info("download tournament: {}", description); + + final Matcher matcher = pattern.matcher(description); + if (!matcher.find()) + { + log.error("Cannot parse tournament description: {}", description); + return; + } + + final LocalDate date = LocalDate.of( // + Integer.parseInt(matcher.group("year")), // + Integer.parseInt(matcher.group("month")), // + Integer.parseInt(matcher.group("dayOfMonth"))); + + for (final WebElement tr : crawler.findElements(By.xpath("//table[@class='tourney_list']/tbody/tr"))) + { + if (hasElement(tr, By.xpath("th"))) + continue; // Header row + + final WebElement archetype = tr.findElement(By.xpath("td[@data-th='Archetype']/a")); + final WebElement player = tr.findElement(By.xpath("td[@data-th='Player']/a")); + final URL downloadUrl = toDownloadUrl(archetype.getAttribute("href")); + + consumer.consume(deckItemFactory.create(downloadUrl, player.getText(), archetype.getText(), date)); + } + } + + private boolean hasElement(final WebElement element, final By by) + { + try + { + element.findElement(by); + return true; + } + catch (final NoSuchElementException e) + { + return false; + } + } + + private URL toDownloadUrl(final String deckUrl) throws MalformedURLException + { + final Matcher matcher = PATTERN_URL_DECK.matcher(deckUrl); + if (!matcher.matches()) + throw new RuntimeException("Cannot parse deck url: " + deckUrl); + + return new URL("https", "www.tcdecks.net", String.format(FORMAT_DOWNLOAD_URL, matcher.group("id"), matcher.group("iddeck"))); + } +} diff --git a/src/main/java/fr/kevincorvisier/mtg/dd/hareruya/HareruyaDecklistDownloader.java b/src/main/java/fr/kevincorvisier/mtg/dd/hareruya/HareruyaDecklistDownloader.java @@ -1,72 +0,0 @@ -package fr.kevincorvisier.mtg.dd.hareruya; - -import java.io.IOException; -import java.net.URL; -import java.time.LocalDate; - -import org.openqa.selenium.By; -import org.openqa.selenium.WebElement; - -import fr.kevincorvisier.mtg.dd.Crawler; -import fr.kevincorvisier.mtg.dd.DecklistConsumer; -import fr.kevincorvisier.mtg.dd.DecklistDownloader; -import fr.kevincorvisier.mtg.dd.model.DeckItem; -import fr.kevincorvisier.mtg.dd.model.DeckItemFactory; -import lombok.RequiredArgsConstructor; - -@RequiredArgsConstructor -public class HareruyaDecklistDownloader implements DecklistDownloader -{ - private final Crawler crawler; - private final DeckItemFactory deckItemFactory; - - @Override - public boolean accept(final URL url) - { - return url.getHost().equals("www.hareruyamtg.com"); - } - - @Override - public void download(final URL url, final DecklistConsumer consumer) - { - crawler.navigateTo(url); - - for (final WebElement element : crawler.findElements(By.className("deckSearch-searchResult__itemWrapper"))) - { - try - { - final DeckItem deck = processDeckElement(element); - consumer.consume(deck); - } - catch (final IOException e) - { - e.printStackTrace(); - } - } - } - - private DeckItem processDeckElement(final WebElement element) throws IOException - { - final String playerName = element.findElement(By.className("deckSearch-searchResult__item__playerName__link")).getText(); - final String deckName = element.findElement(By.xpath(".//li[@class='deckSearch-searchResult__item__deckName']/span[2]")).getText(); - final String date = parseDate(element, By.className("deckSearch-searchResult__item__tournamentDate")); - - // Retrieve the id from the URL, then build the download URL from it - // Format: https://www.hareruyamtg.com/en/deck/361764/show/ - final String href = element.getAttribute("href"); - final int beginIndex = "https://www.hareruyamtg.com/en/deck/".length(); - final int endIndex = href.length() - "/show/".length(); - final long id = Long.parseLong(href.substring(beginIndex, endIndex)); - final URL url = new URL("https://www.hareruyamtg.com/en/deck/download/" + id); - - return deckItemFactory.create(url, playerName, deckName, LocalDate.parse(date)); - } - - private String parseDate(final WebElement element, final By by) - { - String text = element.findElement(by).getText(); - if (text.startsWith("Date：")) - text = text.substring("Date：".length()); - return text.replace('/', '-').replace(" ", ""); - } -} diff --git a/src/main/java/fr/kevincorvisier/mtg/dd/model/DeckItemFactory.java b/src/main/java/fr/kevincorvisier/mtg/dd/model/DeckItemFactory.java @@ -4,13 +4,18 @@ import java.net.URL; import java.time.LocalDate; import java.util.Map; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Service; + import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; @Slf4j +@Service @RequiredArgsConstructor public class DeckItemFactory { + @Value("${replace-archetypes}") private final Map<String, String> replaceArchetypes; public DeckItem create(final URL url, final String player, final String archetype, final LocalDate date) diff --git a/src/main/java/fr/kevincorvisier/mtg/dd/spring/ConversionConfiguration.java b/src/main/java/fr/kevincorvisier/mtg/dd/spring/ConversionConfiguration.java @@ -0,0 +1,22 @@ +package fr.kevincorvisier.mtg.dd.spring; + +import java.util.Set; + +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.core.convert.ConversionService; +import org.springframework.core.convert.support.ConfigurableConversionService; +import org.springframework.core.convert.support.ConversionServiceFactory; +import org.springframework.core.convert.support.DefaultConversionService; + +@Configuration +public class ConversionConfiguration +{ + @Bean + public ConversionService conversionService() + { + final ConfigurableConversionService conversionService = new DefaultConversionService(); + ConversionServiceFactory.registerConverters(Set.of(new StringToMapConverter(conversionService)), conversionService); + return conversionService; + } +} diff --git a/src/main/java/fr/kevincorvisier/mtg/dd/spring/StringToMapConverter.java b/src/main/java/fr/kevincorvisier/mtg/dd/spring/StringToMapConverter.java @@ -0,0 +1,66 @@ +package fr.kevincorvisier.mtg.dd.spring; + +import java.util.Collections; +import java.util.Map; +import java.util.Set; + +import javax.annotation.Nullable; + +import org.springframework.core.CollectionFactory; +import org.springframework.core.convert.ConversionService; +import org.springframework.core.convert.TypeDescriptor; +import org.springframework.core.convert.converter.ConditionalGenericConverter; +import org.springframework.util.StringUtils; + +import lombok.RequiredArgsConstructor; + +@RequiredArgsConstructor +public class StringToMapConverter implements ConditionalGenericConverter +{ + private final ConversionService conversionService; + + @Override + @Nullable + public Set<ConvertiblePair> getConvertibleTypes() + { + return Collections.singleton(new ConvertiblePair(String.class, Map.class)); + } + + @Override + public boolean matches(final TypeDescriptor sourceType, final TypeDescriptor targetType) + { + final TypeDescriptor keyType = targetType.getMapKeyTypeDescriptor(); + final TypeDescriptor valueType = targetType.getMapValueTypeDescriptor(); + + return (keyType == null || conversionService.canConvert(sourceType, keyType)) + && (valueType == null || conversionService.canConvert(sourceType, valueType)); + } + + @Override + @Nullable + public Object convert(@Nullable final Object source, final TypeDescriptor sourceType, final TypeDescriptor targetType) + { + if (source == null) + return null; + + final String string = (String) source; + + final String[] fields = StringUtils.commaDelimitedListToStringArray(string); + final TypeDescriptor keyType = targetType.getMapKeyTypeDescriptor(); + final TypeDescriptor valueType = targetType.getMapValueTypeDescriptor(); + + final Map<Object, Object> target = CollectionFactory.createMap(targetType.getType(), (keyType != null ? keyType.getType() : null), fields.length); + + for (final String field : fields) + { + final String[] keyValue = field.split("\\|", 2); + + final Object key = keyType == null ? keyValue[0].trim() : conversionService.convert(keyValue[0].trim(), sourceType, keyType); + final Object value = valueType == null ? keyValue[1].trim() : conversionService.convert(keyValue[1].trim(), sourceType, valueType); + + target.put(key, value); + } + + return target; + } +} diff --git a/src/main/java/fr/kevincorvisier/mtg/dd/tcdecks/TcdecksDecklistDownloader.java b/src/main/java/fr/kevincorvisier/mtg/dd/tcdecks/TcdecksDecklistDownloader.java @@ -1,214 +0,0 @@ -package fr.kevincorvisier.mtg.dd.tcdecks; - -import java.io.IOException; -import java.net.MalformedURLException; -import java.net.URL; -import java.time.LocalDate; -import java.time.format.DateTimeFormatter; -import java.util.ArrayList; -import java.util.List; -import java.util.Properties; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -import org.openqa.selenium.By; -import org.openqa.selenium.NoSuchElementException; -import org.openqa.selenium.WebElement; - -import fr.kevincorvisier.mtg.dd.Crawler; -import fr.kevincorvisier.mtg.dd.DecklistConsumer; -import fr.kevincorvisier.mtg.dd.DecklistDownloader; -import fr.kevincorvisier.mtg.dd.model.DeckItemFactory; -import lombok.RequiredArgsConstructor; -import lombok.extern.slf4j.Slf4j; - -@Slf4j -@RequiredArgsConstructor -public class TcdecksDecklistDownloader implements DecklistDownloader -{ - private static final Pattern PATTERN_URL_FORMAT = Pattern.compile("https://www\\.tcdecks\\.net/format\\.php\\?format=(?:[A-Za-z])+"); - private static final Pattern PATTERN_URL_ARCHETYPE = Pattern - .compile("https://www\\.tcdecks\\.net/archetype\\.php\\?archetype=(?<archetype>[A-Za-z]+)&format=[A-Za-z]+"); - private static final Pattern PATTERN_URL_DECK = Pattern.compile("https://www.tcdecks.net/deck\\.php\\?id=(?<id>\\d+)&iddeck=(?<iddeck>\\d+)"); - - private static final Pattern PATTERN_POSITION = Pattern.compile("\\d+ of (?<players>\\d+)"); - - private static final Pattern pattern = Pattern.compile("Date: (?<dayOfMonth>\\d{2})\\/(?<month>\\d{2})\\/(?<year>\\d{4})"); - private static final Pattern PATTERN_PLAYERS = Pattern.compile("(?<players>\\d+) Players"); - - private static final String FORMAT_DOWNLOAD_URL = "/download.php?ext=txt&id=%1$s&iddeck=%2$s"; - - private final Crawler crawler; - private final DeckItemFactory deckItemFactory; - private final int tournamentMinPlayers; - - public TcdecksDecklistDownloader(final Crawler crawler, final DeckItemFactory deckItemFactory, final Properties props) - { - this.crawler = crawler; - this.deckItemFactory = deckItemFactory; - tournamentMinPlayers = Integer.parseInt(props.getProperty("tcdecks.tournament.players.min", "0")); - } - - @Override - public boolean accept(final URL url) - { - return url.getHost().equals("www.tcdecks.net"); - } - - @Override - public void download(final URL url, final DecklistConsumer consumer) throws IOException - { - final String urlStr = url.toString(); - - if (PATTERN_URL_FORMAT.matcher(urlStr).matches()) - { - downloadFormat(url, consumer); - return; - } - - final Matcher matcher = PATTERN_URL_ARCHETYPE.matcher(urlStr); - if (matcher.matches()) - { - final String archetype = matcher.group("archetype"); - downloadArchetype(url, archetype, consumer); - } - } - - private void downloadFormat(final URL url, final DecklistConsumer consumer) throws IOException - { - URL next = url; - - do - { - crawler.navigateTo(next); - - final List<URL> tournaments = new ArrayList<>(); - - for (final WebElement tr : crawler.findElements(By.xpath("//table[@class='tourney_list']/tbody/tr"))) - { - if (hasElement(tr, By.xpath("th"))) - continue; // Header row - - final WebElement tournamentName = tr.findElement(By.xpath("td[@data-th='Tournament Name']/a")); - final String players = tr.findElement(By.xpath("td[@data-th='Players']/a")).getText(); - - if ("Unknown".equals(players)) - continue; // Skip tournaments with no player count - - final Matcher matcher = PATTERN_PLAYERS.matcher(players); - if (!matcher.matches()) - { - log.error("Cannot parse players column: {}", players); - return; - } - - final int nbPlayers = Integer.parseInt(matcher.group("players")); - if (nbPlayers < tournamentMinPlayers) - { - log.info("Skip {}: less than {} players ({} players)", tournamentName.getText(), tournamentMinPlayers, nbPlayers); - continue; // Skip small tournaments - } - - tournaments.add(new URL(tournamentName.getAttribute("href"))); - } - - next = new URL(crawler.findElement(By.xpath("//a[text()='Next']")).getAttribute("href")); - - for (final URL tournament : tournaments) - downloadTournament(tournament, consumer); - } while (consumer.capacity() > 0); - } - - private void downloadArchetype(final URL url, final String archetype, final DecklistConsumer consumer) throws IOException - { - URL next = url; - - do - { - crawler.navigateTo(next); - - next = new URL(crawler.findElement(By.xpath("//a[text()='Next']")).getAttribute("href")); - - for (final WebElement tr : crawler.findElements(By.xpath("//table[@class='tourney_list']/tbody/tr"))) - { - if (hasElement(tr, By.xpath("th"))) - continue; // Header row - - final WebElement deckName = tr.findElement(By.xpath("td[@data-th='Deck Name']/a")); - final WebElement player = tr.findElement(By.xpath("td[@data-th='Player']/a")); - final WebElement date = tr.findElement(By.xpath("td[@data-th='Date']/a")); - final WebElement position = tr.findElement(By.xpath("td[@data-th='Position']/a")); - final URL downloadUrl = toDownloadUrl(deckName.getAttribute("href")); - - final Matcher matcher = PATTERN_POSITION.matcher(position.getText()); - if (matcher.matches()) - { - final int nbPlayers = Integer.parseInt(matcher.group("players")); - if (nbPlayers < tournamentMinPlayers) - { - log.info("Skip {}: less than {} players ({} players)", deckName.getText(), tournamentMinPlayers, nbPlayers); - continue; // Skip small tournaments - } - } - - consumer.consume(deckItemFactory.create(downloadUrl, player.getText(), archetype, - LocalDate.parse(date.getText(), DateTimeFormatter.ofPattern("dd/MM/yyyy")))); - } - } while (consumer.capacity() > 0); - } - - private void downloadTournament(final URL url, final DecklistConsumer consumer) throws IOException - { - crawler.navigateTo(url); - - final String description = crawler.findElement(By.xpath("//article[@class='span8']/fieldset/legend/h5")).getText(); - - log.info("download tournament: {}", description); - - final Matcher matcher = pattern.matcher(description); - if (!matcher.find()) - { - log.error("Cannot parse tournament description: {}", description); - return; - } - - final LocalDate date = LocalDate.of( // - Integer.parseInt(matcher.group("year")), // - Integer.parseInt(matcher.group("month")), // - Integer.parseInt(matcher.group("dayOfMonth"))); - - for (final WebElement tr : crawler.findElements(By.xpath("//table[@class='tourney_list']/tbody/tr"))) - { - if (hasElement(tr, By.xpath("th"))) - continue; // Header row - - final WebElement archetype = tr.findElement(By.xpath("td[@data-th='Archetype']/a")); - final WebElement player = tr.findElement(By.xpath("td[@data-th='Player']/a")); - final URL downloadUrl = toDownloadUrl(archetype.getAttribute("href")); - - consumer.consume(deckItemFactory.create(downloadUrl, player.getText(), archetype.getText(), date)); - } - } - - private boolean hasElement(final WebElement element, final By by) - { - try - { - element.findElement(by); - return true; - } - catch (final NoSuchElementException e) - { - return false; - } - } - - private URL toDownloadUrl(final String deckUrl) throws MalformedURLException - { - final Matcher matcher = PATTERN_URL_DECK.matcher(deckUrl); - if (!matcher.matches()) - throw new RuntimeException("Cannot parse deck url: " + deckUrl); - - return new URL("https", "www.tcdecks.net", String.format(FORMAT_DOWNLOAD_URL, matcher.group("id"), matcher.group("iddeck"))); - } -} diff --git a/src/main/java/fr/kevincorvisier/mtg/dd/validation/AiCards.java b/src/main/java/fr/kevincorvisier/mtg/dd/validation/AiCards.java @@ -6,9 +6,12 @@ import java.io.InputStreamReader; import java.util.Collection; import java.util.HashSet; +import org.springframework.stereotype.Service; + import lombok.extern.slf4j.Slf4j; @Slf4j +@Service public class AiCards { private final Collection<String> playableCards; diff --git a/src/main/java/fr/kevincorvisier/mtg/dd/validation/DeckValidator.java b/src/main/java/fr/kevincorvisier/mtg/dd/validation/DeckValidator.java @@ -1,19 +1,39 @@ package fr.kevincorvisier.mtg.dd.validation; +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; import java.util.Collection; +import java.util.HashSet; import java.util.Map.Entry; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Service; + import fr.kevincorvisier.mtg.dd.model.DeckItem; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; @Slf4j +@Service @RequiredArgsConstructor public class DeckValidator { - private final Collection<String> banlist; + private final Collection<String> banlist = new HashSet<>(); private final AiCards aiCards; + public void setBanlistFileNames(@Value("${banlists}") final Collection<String> banlistFileNames) throws IOException + { + for (String banlistFileName : banlistFileNames) + { + banlistFileName = banlistFileName.trim(); + if (banlistFileName.isEmpty()) + continue; + + readBanlist(banlistFileName); + } + } + public boolean validate(final DeckItem deck, final boolean onlyAiPlayableCards) { int cardCount = 0; @@ -44,4 +64,19 @@ public class DeckValidator return true; } + + private void readBanlist(final String name) throws IOException + { + try (BufferedReader reader = new BufferedReader(new InputStreamReader(getClass().getClassLoader().getResourceAsStream(name)))) + { + String line; + while ((line = reader.readLine()) != null) + { + line = line.trim(); + if (line.isEmpty()) + continue; + banlist.add(line); + } + } + } } diff --git a/src/main/java/fr/kevincorvisier/mtg/dd/validation/DeckValidatorFactory.java b/src/main/java/fr/kevincorvisier/mtg/dd/validation/DeckValidatorFactory.java @@ -1,67 +0,0 @@ -package fr.kevincorvisier.mtg.dd.validation; - -import java.io.BufferedReader; -import java.io.IOException; -import java.io.InputStreamReader; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Map; -import java.util.Properties; -import java.util.Set; - -import lombok.AccessLevel; -import lombok.NoArgsConstructor; - -@NoArgsConstructor(access = AccessLevel.PRIVATE) // static class -public class DeckValidatorFactory -{ - public static Map<String, DeckValidator> create(final Iterable<String> formats, final Properties props) throws IOException - { - final AiCards aiCards = new AiCards(); - - final Map<String, Collection<String>> banlistByName = new HashMap<>(); - final Map<String, DeckValidator> validatorByFormat = new HashMap<>(); - - for (final String format : formats) - { - final Set<String> formatBanlist = new HashSet<>(); - - for (String banlistName : props.getProperty("formats." + format + ".banlists", "").split(",")) - { - banlistName = banlistName.trim(); - if (banlistName.isEmpty()) - continue; - - Collection<String> banlist = banlistByName.get(banlistName); - if (banlist == null) - banlistByName.put(banlistName, banlist = readBanlist(banlistName)); - formatBanlist.addAll(banlist); - } - - validatorByFormat.put(format, new DeckValidator(Collections.unmodifiableSet(formatBanlist), aiCards)); - } - - return validatorByFormat; - } - - private static Set<String> readBanlist(final String name) throws IOException - { - final Set<String> banlist = new HashSet<>(); - - try (BufferedReader reader = new BufferedReader(new InputStreamReader(DeckValidatorFactory.class.getClassLoader().getResourceAsStream(name)))) - { - String line; - while ((line = reader.readLine()) != null) - { - line = line.trim(); - if (line.isEmpty()) - continue; - banlist.add(line); - } - } - - return Collections.unmodifiableSet(banlist); - } -} diff --git a/src/main/packaged-resources/cfg/application.properties b/src/main/packaged-resources/cfg/application.properties @@ -1,57 +1,4 @@ - cache.database=/home/kebi/Documents/mtg/cache.db - ignore.players=Moreau,\u30E2\u30ED\u30FC replace-archetypes=Mono Red[MiddleSchool]|Burn,Sligh|Burn,Other|Rogue,Goblin[Middle School]|Goblins - -# formats=example1-ms-opponents,example1-pm-opponents -# tcdecks.tournament.players.min=64 - -# formats=example1-initial-population -# tcdecks.tournament.players.min=128 - -formats=pauper-meta,standard-meta - -formats.example1-ms-opponents.banlists=banlist_custom.txt -formats.example1-ms-opponents.limit=13 -formats.example1-ms-opponents.archetype-limit=1 -formats.example1-ms-opponents.only-ai-playable-cards=true -formats.example1-ms-opponents.sources.0=https://www.hareruyamtg.com/en/deck/result?pageSize=100&formats[11]=11&eventName=middle&public_status=public&grades=champion -formats.example1-ms-opponents.sources.1=https://www.hareruyamtg.com/en/deck/result?pageSize=100&formats[11]=11&eventName=middle&public_status=public&grades=top8 -formats.example1-ms-opponents.sources.2=https://www.hareruyamtg.com/en/deck/result?pageSize=100&formats[11]=11&eventName=middle&public_status=public&grades=top8&page=2 -formats.example1-ms-opponents.sources.3=https://www.hareruyamtg.com/en/deck/result?pageSize=100&formats[11]=11&eventName=middle&public_status=public -formats.example1-ms-opponents.sources.4=https://www.hareruyamtg.com/en/deck/result?pageSize=100&formats[11]=11&eventName=middle&public_status=public&page=2 -formats.example1-ms-opponents.sources.5=https://www.hareruyamtg.com/en/deck/result?pageSize=100&formats[11]=11&eventName=middle&public_status=public&page=3 -formats.example1-ms-opponents.sources.6=https://www.hareruyamtg.com/en/deck/result?pageSize=100&formats[11]=11&eventName=middle&public_status=public&page=4 -formats.example1-ms-opponents.output-dir=/home/kebi/git/repositories/mtg-genetic-deckbuilding/src/main/packaged-resources/cfg/example1/ms-opponents - -formats.example1-pm-opponents.banlists=banlist_custom.txt -formats.example1-pm-opponents.limit=12 -formats.example1-pm-opponents.archetype-limit=1 -formats.example1-pm-opponents.only-ai-playable-cards=true -formats.example1-pm-opponents.sources.0=https://www.tcdecks.net/format.php?format=Premodern -formats.example1-pm-opponents.output-dir=/home/kebi/git/repositories/mtg-genetic-deckbuilding/src/main/packaged-resources/cfg/example1/pm-opponents2 - -formats.example1-initial-population.banlists=banlist_ec.txt,banlist_custom.txt -formats.example1-initial-population.limit=20 -formats.example1-initial-population.archetype-limit=20 -formats.example1-initial-population.only-ai-playable-cards=true -formats.example1-initial-population.sources.0=https://www.hareruyamtg.com/en/deck/result?pageSize=100&formats[11]=11&eventName=middle&public_status=public&grades=top8&archetypeIds=5841,5885,6440 -formats.example1-initial-population.sources.1=https://www.tcdecks.net/archetype.php?archetype=Burn&format=Premodern -formats.example1-initial-population.output-dir=/home/kebi/git/repositories/mtg-genetic-deckbuilding/src/main/packaged-resources/cfg/example1/initial-population - -formats.pauper-meta.banlists=banlist_custom.txt -formats.pauper-meta.limit=100 -formats.pauper-meta.archetype-limit=100 -formats.pauper-meta.only-ai-playable-cards=true -formats.pauper-meta.sources.0=https://www.hareruyamtg.com/en/deck/result?pageSize=100&formats[8]=8&public_status=public -formats.pauper-meta.output-dir=/home/kebi/Documents/mtg/pauper/meta - -formats.standard-meta.banlists=banlist_custom.txt -formats.standard-meta.limit=100 -formats.standard-meta.archetype-limit=100 -formats.standard-meta.only-ai-playable-cards=true -formats.standard-meta.sources.0=https://www.hareruyamtg.com/en/deck/result?pageSize=100&formats[1]=1&public_status=public -formats.standard-meta.output-dir=/home/kebi/Documents/mtg/standard/meta - diff --git a/src/main/packaged-resources/cfg/config-available/example1-initial-population.properties b/src/main/packaged-resources/cfg/config-available/example1-initial-population.properties @@ -0,0 +1,8 @@ +banlists=banlist_ec.txt,banlist_custom.txt +limit=20 +archetype-limit=20 +only-ai-playable-cards=true +sources=https://www.hareruyamtg.com/en/deck/result?pageSize=100&formats[11]=11&eventName=middle&public_status=public&grades=top8&archetypeIds=5841,5885,6440 | \ + https://www.tcdecks.net/archetype.php?archetype=Burn&format=Premodern +output-dir=/home/kebi/git/repositories/mtg-genetic-deckbuilding/src/main/packaged-resources/cfg/example1/initial-population +tcdecks.tournament.players.min=128 +\ No newline at end of file diff --git a/src/main/packaged-resources/cfg/config-available/example1-ms-opponents.properties b/src/main/packaged-resources/cfg/config-available/example1-ms-opponents.properties @@ -0,0 +1,13 @@ +banlists=banlist_custom.txt +limit=13 +archetype-limit=1 +only-ai-playable-cards=true +sources=https://www.hareruyamtg.com/en/deck/result?pageSize=100&formats[11]=11&eventName=middle&public_status=public&grades=champion | \ + https://www.hareruyamtg.com/en/deck/result?pageSize=100&formats[11]=11&eventName=middle&public_status=public&grades=top8 | \ + https://www.hareruyamtg.com/en/deck/result?pageSize=100&formats[11]=11&eventName=middle&public_status=public&grades=top8&page=2 | \ + https://www.hareruyamtg.com/en/deck/result?pageSize=100&formats[11]=11&eventName=middle&public_status=public | \ + https://www.hareruyamtg.com/en/deck/result?pageSize=100&formats[11]=11&eventName=middle&public_status=public&page=2 | \ + https://www.hareruyamtg.com/en/deck/result?pageSize=100&formats[11]=11&eventName=middle&public_status=public&page=3 | \ + https://www.hareruyamtg.com/en/deck/result?pageSize=100&formats[11]=11&eventName=middle&public_status=public&page=4 +output-dir=/home/kebi/git/repositories/mtg-genetic-deckbuilding/src/main/packaged-resources/cfg/example1/ms-opponents +tcdecks.tournament.players.min=64 +\ No newline at end of file diff --git a/src/main/packaged-resources/cfg/config-available/example1-pm-opponents.properties b/src/main/packaged-resources/cfg/config-available/example1-pm-opponents.properties @@ -0,0 +1,7 @@ +banlists=banlist_custom.txt +limit=12 +archetype-limit=1 +only-ai-playable-cards=true +sources=https://www.tcdecks.net/format.php?format=Premodern +output-dir=/home/kebi/git/repositories/mtg-genetic-deckbuilding/src/main/packaged-resources/cfg/example1/pm-opponents2 +tcdecks.tournament.players.min=64 +\ No newline at end of file diff --git a/src/main/packaged-resources/cfg/config-available/pauper-meta.properties b/src/main/packaged-resources/cfg/config-available/pauper-meta.properties @@ -0,0 +1,6 @@ +banlists=banlist_custom.txt +limit=100 +archetype-limit=100 +only-ai-playable-cards=true +sources=https://www.hareruyamtg.com/en/deck/result?pageSize=100&formats[8]=8&public_status=public +output-dir=/home/kebi/Documents/mtg/pauper/meta +\ No newline at end of file diff --git a/src/main/packaged-resources/cfg/config-available/standard-meta.properties b/src/main/packaged-resources/cfg/config-available/standard-meta.properties @@ -0,0 +1,6 @@ +banlists=banlist_custom.txt +limit=100 +archetype-limit=100 +only-ai-playable-cards=true +sources=https://www.hareruyamtg.com/en/deck/result?pageSize=100&formats[1]=1&public_status=public +output-dir=/home/kebi/Documents/mtg/standard/meta +\ No newline at end of file