commit 2c1889c1f128f4f50f44a0155807025175c9333d
parent 12f8aebc3dea18a2d197e08d17ece47a32fa4f46
Author: Kevin Corvisier <git@kevincorvisier.fr>
Date: Sat, 18 Jan 2025 21:06:44 +0900
Hareruya decklist downloader: handle pagination
Diffstat:
4 files changed, 48 insertions(+), 22 deletions(-)
diff --git a/src/main/java/fr/kevincorvisier/mtg/dd/downloaders/HareruyaDecklistDownloader.java b/src/main/java/fr/kevincorvisier/mtg/dd/downloaders/HareruyaDecklistDownloader.java
@@ -1,10 +1,12 @@
package fr.kevincorvisier.mtg.dd.downloaders;
import java.io.IOException;
+import java.net.MalformedURLException;
import java.net.URL;
import java.time.LocalDate;
import org.openqa.selenium.By;
+import org.openqa.selenium.NoSuchElementException;
import org.openqa.selenium.WebElement;
import org.springframework.stereotype.Service;
@@ -31,25 +33,39 @@ public class HareruyaDecklistDownloader implements DecklistDownloader
}
@Override
- public void download(final URL url)
+ public void download(final URL url) throws MalformedURLException
{
- crawler.navigateTo(url);
+ URL next = url;
- for (final WebElement element : crawler.findElements(By.className("deckSearch-searchResult__itemWrapper")))
+ do
{
- try
+ crawler.navigateTo(next);
+
+ for (final WebElement element : crawler.findElements(By.className("deckSearch-searchResult__itemWrapper")))
{
- final DeckMetadata deck = processDeckElement(element);
- consumers.process(deck);
+ try
+ {
+ final DeckMetadata deck = processDeckElement(element);
+ consumers.process(deck);
+
+ if (stopCondition.capacity() <= 0)
+ return;
+ }
+ catch (final IOException e)
+ {
+ e.printStackTrace();
+ }
+ }
- if (stopCondition.capacity() <= 0)
- return;
+ try
+ {
+ next = new URL(crawler.findElement(By.xpath("//div[@class='searchResultPagenation']/a[text()='>']")).getAttribute("href"));
}
- catch (final IOException e)
+ catch (final NoSuchElementException e)
{
- e.printStackTrace();
+ next = null;
}
- }
+ } while (stopCondition.capacity() > 0 && next != null);
}
private DeckMetadata processDeckElement(final WebElement element) throws IOException
diff --git a/src/main/packaged-resources/cfg/config-available/example1-ms-opponents.properties b/src/main/packaged-resources/cfg/config-available/example1-ms-opponents.properties
@@ -2,11 +2,5 @@ limit=15
archetype-limit=1
sources=https://www.hareruyamtg.com/en/deck/result?pageSize=100&formats[11]=11&eventName=middle&public_status=public&grades=champion | \
https://www.hareruyamtg.com/en/deck/result?pageSize=100&formats[11]=11&eventName=middle&public_status=public&grades=top8 | \
- https://www.hareruyamtg.com/en/deck/result?pageSize=100&formats[11]=11&eventName=middle&public_status=public&grades=top8&page=2 | \
- https://www.hareruyamtg.com/en/deck/result?pageSize=100&formats[11]=11&eventName=middle&public_status=public&grades=top8&page=3 | \
- https://www.hareruyamtg.com/en/deck/result?pageSize=100&formats[11]=11&eventName=middle&public_status=public | \
- https://www.hareruyamtg.com/en/deck/result?pageSize=100&formats[11]=11&eventName=middle&public_status=public&page=2 | \
- https://www.hareruyamtg.com/en/deck/result?pageSize=100&formats[11]=11&eventName=middle&public_status=public&page=3 | \
- https://www.hareruyamtg.com/en/deck/result?pageSize=100&formats[11]=11&eventName=middle&public_status=public&page=4 | \
- https://www.hareruyamtg.com/en/deck/result?pageSize=100&formats[11]=11&eventName=middle&public_status=public&page=5
+ https://www.hareruyamtg.com/en/deck/result?pageSize=100&formats[11]=11&eventName=middle&public_status=public
output-dir=/home/kebi/git/repositories/mtg-genetic-deckbuilding/src/main/packaged-resources/cfg/example1/ms-opponents
\ No newline at end of file
diff --git a/src/main/packaged-resources/cfg/config-available/example1-pms-card-pool.properties b/src/main/packaged-resources/cfg/config-available/example1-pms-card-pool.properties
@@ -1,10 +1,6 @@
limit=1000
banlists=banlist_premodern.txt,banlist_middleschool.txt,error-prone-cards.txt
sources=https://www.hareruyamtg.com/en/deck/result?pageSize=100&formats[11]=11&eventName=middle&public_status=public | \
- https://www.hareruyamtg.com/en/deck/result?pageSize=100&formats[11]=11&eventName=middle&public_status=public&page=2 | \
- https://www.hareruyamtg.com/en/deck/result?pageSize=100&formats[11]=11&eventName=middle&public_status=public&page=3 | \
- https://www.hareruyamtg.com/en/deck/result?pageSize=100&formats[11]=11&eventName=middle&public_status=public&page=4 | \
- https://www.hareruyamtg.com/en/deck/result?pageSize=100&formats[11]=11&eventName=middle&public_status=public&page=5 | \
https://www.tcdecks.net/format.php?format=Premodern
output-dir=/tmp/pms
card-pool.enabled=true
diff --git a/src/test/java/fr/kevincorvisier/mtg/dd/downloaders/HareruyaDecklistDownloaderTest.java b/src/test/java/fr/kevincorvisier/mtg/dd/downloaders/HareruyaDecklistDownloaderTest.java
@@ -0,0 +1,20 @@
+package fr.kevincorvisier.mtg.dd.downloaders;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+import java.net.MalformedURLException;
+import java.net.URL;
+
+import org.junit.jupiter.api.Test;
+
+class HareruyaDecklistDownloaderTest
+{
+ @Test
+ void accept() throws MalformedURLException
+ {
+ final HareruyaDecklistDownloader tested = new HareruyaDecklistDownloader(null, null, null, null);
+
+ assertEquals(true, tested.accept(
+ new URL("https://www.hareruyamtg.com/en/deck/result?pageSize=100&formats[11]=11&eventName=middle&public_status=public&grades=champion")));
+ }
+}
