commit da423c009c81d1952f49a62cd329388e68d757b8 Author: Kevin Corvisier <git@kevincorvisier.fr> Date: Thu, 20 Mar 2025 11:01:34 +0900 Initial commit, synchronization of scryfall bulk data into a sqlite database Diffstat:
23 files changed, 1428 insertions(+), 0 deletions(-)
diff --git a/.gitignore b/.gitignore @@ -0,0 +1,84 @@ +# +# Eclipse +# + +.metadata +bin/ +tmp/ +*.tmp +*.bak +*.swp +*~.nib +local.properties +.settings/ +.loadpath +.recommenders + +# External tool builders +.externalToolBuilders/ + +# PyDev specific (Python IDE for Eclipse) +*.pydevproject + +# CDT-specific (C/C++ Development Tooling) +.cproject + +# CDT- autotools +.autotools + +# Java annotation processor (APT) +.factorypath + +# PDT-specific (PHP Development Tools) +.buildpath + +# sbteclipse plugin +.target + +# Tern plugin +.tern-project + +# TeXlipse plugin +.texlipse + +# STS (Spring Tool Suite) +.springBeans + +# Code Recommenders +.recommenders/ + +# Annotation Processing +.apt_generated/ +.apt_generated_test/ + +# Scala IDE specific (Scala & Java development for Eclipse) +.cache-main +.scala_dependencies +.worksheet + +# Uncomment this line if you wish to ignore the project description file. +# Typically, this file would be tracked if it contains build/dependency configurations: +#.project + + +# +# Maven +# + +target/ +pom.xml.tag +pom.xml.releaseBackup +pom.xml.versionsBackup +pom.xml.next +release.properties +dependency-reduced-pom.xml +buildNumber.properties +.mvn/timing.properties +# https://github.com/takari/maven-wrapper#usage-without-binary-jar +.mvn/wrapper/maven-wrapper.jar + +# Eclipse m2e generated files +# Eclipse Core +.project +# JDT-specific (Eclipse Java Development Tools) +.classpath diff --git a/.mvn/wrapper/maven-wrapper.properties b/.mvn/wrapper/maven-wrapper.properties @@ -0,0 +1,19 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +wrapperVersion=3.3.2 +distributionType=only-script +distributionUrl=https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.9.9/apache-maven-3.9.9-bin.zip diff --git a/lombok.config b/lombok.config @@ -0,0 +1,7 @@ +# Copy annotations from fields to the generated constructors +lombok.copyableAnnotations += jakarta.annotation.Nullable +lombok.copyableAnnotations += jakarta.validation.constraints.NotNull +lombok.copyableAnnotations += org.springframework.beans.factory.annotation.Autowired +lombok.copyableAnnotations += org.springframework.beans.factory.annotation.Qualifier +lombok.copyableAnnotations += org.springframework.beans.factory.annotation.Value +lombok.addNullAnnotations = CUSTOM:jakarta.validation.constraints.NotNull:jakarta.annotation.Nullable +\ No newline at end of file diff --git a/mvnw b/mvnw @@ -0,0 +1,259 @@ +#!/bin/sh +# ---------------------------------------------------------------------------- +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# ---------------------------------------------------------------------------- + +# ---------------------------------------------------------------------------- +# Apache Maven Wrapper startup batch script, version 3.3.2 +# +# Optional ENV vars +# ----------------- +# JAVA_HOME - location of a JDK home dir, required when download maven via java source +# MVNW_REPOURL - repo url base for downloading maven distribution +# MVNW_USERNAME/MVNW_PASSWORD - user and password for downloading maven +# MVNW_VERBOSE - true: enable verbose log; debug: trace the mvnw script; others: silence the output +# ---------------------------------------------------------------------------- + +set -euf +[ "${MVNW_VERBOSE-}" != debug ] || set -x + +# OS specific support. +native_path() { printf %s\\n "$1"; } +case "$(uname)" in +CYGWIN* | MINGW*) + [ -z "${JAVA_HOME-}" ] || JAVA_HOME="$(cygpath --unix "$JAVA_HOME")" + native_path() { cygpath --path --windows "$1"; } + ;; +esac + +# set JAVACMD and JAVACCMD +set_java_home() { + # For Cygwin and MinGW, ensure paths are in Unix format before anything is touched + if [ -n "${JAVA_HOME-}" ]; then + if [ -x "$JAVA_HOME/jre/sh/java" ]; then + # IBM's JDK on AIX uses strange locations for the executables + JAVACMD="$JAVA_HOME/jre/sh/java" + JAVACCMD="$JAVA_HOME/jre/sh/javac" + else + JAVACMD="$JAVA_HOME/bin/java" + JAVACCMD="$JAVA_HOME/bin/javac" + + if [ ! -x "$JAVACMD" ] || [ ! -x "$JAVACCMD" ]; then + echo "The JAVA_HOME environment variable is not defined correctly, so mvnw cannot run." >&2 + echo "JAVA_HOME is set to \"$JAVA_HOME\", but \"\$JAVA_HOME/bin/java\" or \"\$JAVA_HOME/bin/javac\" does not exist." >&2 + return 1 + fi + fi + else + JAVACMD="$( + 'set' +e + 'unset' -f command 2>/dev/null + 'command' -v java + )" || : + JAVACCMD="$( + 'set' +e + 'unset' -f command 2>/dev/null + 'command' -v javac + )" || : + + if [ ! -x "${JAVACMD-}" ] || [ ! -x "${JAVACCMD-}" ]; then + echo "The java/javac command does not exist in PATH nor is JAVA_HOME set, so mvnw cannot run." >&2 + return 1 + fi + fi +} + +# hash string like Java String::hashCode +hash_string() { + str="${1:-}" h=0 + while [ -n "$str" ]; do + char="${str%"${str#?}"}" + h=$(((h * 31 + $(LC_CTYPE=C printf %d "'$char")) % 4294967296)) + str="${str#?}" + done + printf %x\\n $h +} + +verbose() { :; } +[ "${MVNW_VERBOSE-}" != true ] || verbose() { printf %s\\n "${1-}"; } + +die() { + printf %s\\n "$1" >&2 + exit 1 +} + +trim() { + # MWRAPPER-139: + # Trims trailing and leading whitespace, carriage returns, tabs, and linefeeds. + # Needed for removing poorly interpreted newline sequences when running in more + # exotic environments such as mingw bash on Windows. + printf "%s" "${1}" | tr -d '[:space:]' +} + +# parse distributionUrl and optional distributionSha256Sum, requires .mvn/wrapper/maven-wrapper.properties +while IFS="=" read -r key value; do + case "${key-}" in + distributionUrl) distributionUrl=$(trim "${value-}") ;; + distributionSha256Sum) distributionSha256Sum=$(trim "${value-}") ;; + esac +done <"${0%/*}/.mvn/wrapper/maven-wrapper.properties" +[ -n "${distributionUrl-}" ] || die "cannot read distributionUrl property in ${0%/*}/.mvn/wrapper/maven-wrapper.properties" + +case "${distributionUrl##*/}" in +maven-mvnd-*bin.*) + MVN_CMD=mvnd.sh _MVNW_REPO_PATTERN=/maven/mvnd/ + case "${PROCESSOR_ARCHITECTURE-}${PROCESSOR_ARCHITEW6432-}:$(uname -a)" in + *AMD64:CYGWIN* | *AMD64:MINGW*) distributionPlatform=windows-amd64 ;; + :Darwin*x86_64) distributionPlatform=darwin-amd64 ;; + :Darwin*arm64) distributionPlatform=darwin-aarch64 ;; + :Linux*x86_64*) distributionPlatform=linux-amd64 ;; + *) + echo "Cannot detect native platform for mvnd on $(uname)-$(uname -m), use pure java version" >&2 + distributionPlatform=linux-amd64 + ;; + esac + distributionUrl="${distributionUrl%-bin.*}-$distributionPlatform.zip" + ;; +maven-mvnd-*) MVN_CMD=mvnd.sh _MVNW_REPO_PATTERN=/maven/mvnd/ ;; +*) MVN_CMD="mvn${0##*/mvnw}" _MVNW_REPO_PATTERN=/org/apache/maven/ ;; +esac + +# apply MVNW_REPOURL and calculate MAVEN_HOME +# maven home pattern: ~/.m2/wrapper/dists/{apache-maven-<version>,maven-mvnd-<version>-<platform>}/<hash> +[ -z "${MVNW_REPOURL-}" ] || distributionUrl="$MVNW_REPOURL$_MVNW_REPO_PATTERN${distributionUrl#*"$_MVNW_REPO_PATTERN"}" +distributionUrlName="${distributionUrl##*/}" +distributionUrlNameMain="${distributionUrlName%.*}" +distributionUrlNameMain="${distributionUrlNameMain%-bin}" +MAVEN_USER_HOME="${MAVEN_USER_HOME:-${HOME}/.m2}" +MAVEN_HOME="${MAVEN_USER_HOME}/wrapper/dists/${distributionUrlNameMain-}/$(hash_string "$distributionUrl")" + +exec_maven() { + unset MVNW_VERBOSE MVNW_USERNAME MVNW_PASSWORD MVNW_REPOURL || : + exec "$MAVEN_HOME/bin/$MVN_CMD" "$@" || die "cannot exec $MAVEN_HOME/bin/$MVN_CMD" +} + +if [ -d "$MAVEN_HOME" ]; then + verbose "found existing MAVEN_HOME at $MAVEN_HOME" + exec_maven "$@" +fi + +case "${distributionUrl-}" in +*?-bin.zip | *?maven-mvnd-?*-?*.zip) ;; +*) die "distributionUrl is not valid, must match *-bin.zip or maven-mvnd-*.zip, but found '${distributionUrl-}'" ;; +esac + +# prepare tmp dir +if TMP_DOWNLOAD_DIR="$(mktemp -d)" && [ -d "$TMP_DOWNLOAD_DIR" ]; then + clean() { rm -rf -- "$TMP_DOWNLOAD_DIR"; } + trap clean HUP INT TERM EXIT +else + die "cannot create temp dir" +fi + +mkdir -p -- "${MAVEN_HOME%/*}" + +# Download and Install Apache Maven +verbose "Couldn't find MAVEN_HOME, downloading and installing it ..." +verbose "Downloading from: $distributionUrl" +verbose "Downloading to: $TMP_DOWNLOAD_DIR/$distributionUrlName" + +# select .zip or .tar.gz +if ! command -v unzip >/dev/null; then + distributionUrl="${distributionUrl%.zip}.tar.gz" + distributionUrlName="${distributionUrl##*/}" +fi + +# verbose opt +__MVNW_QUIET_WGET=--quiet __MVNW_QUIET_CURL=--silent __MVNW_QUIET_UNZIP=-q __MVNW_QUIET_TAR='' +[ "${MVNW_VERBOSE-}" != true ] || __MVNW_QUIET_WGET='' __MVNW_QUIET_CURL='' __MVNW_QUIET_UNZIP='' __MVNW_QUIET_TAR=v + +# normalize http auth +case "${MVNW_PASSWORD:+has-password}" in +'') MVNW_USERNAME='' MVNW_PASSWORD='' ;; +has-password) [ -n "${MVNW_USERNAME-}" ] || MVNW_USERNAME='' MVNW_PASSWORD='' ;; +esac + +if [ -z "${MVNW_USERNAME-}" ] && command -v wget >/dev/null; then + verbose "Found wget ... using wget" + wget ${__MVNW_QUIET_WGET:+"$__MVNW_QUIET_WGET"} "$distributionUrl" -O "$TMP_DOWNLOAD_DIR/$distributionUrlName" || die "wget: Failed to fetch $distributionUrl" +elif [ -z "${MVNW_USERNAME-}" ] && command -v curl >/dev/null; then + verbose "Found curl ... using curl" + curl ${__MVNW_QUIET_CURL:+"$__MVNW_QUIET_CURL"} -f -L -o "$TMP_DOWNLOAD_DIR/$distributionUrlName" "$distributionUrl" || die "curl: Failed to fetch $distributionUrl" +elif set_java_home; then + verbose "Falling back to use Java to download" + javaSource="$TMP_DOWNLOAD_DIR/Downloader.java" + targetZip="$TMP_DOWNLOAD_DIR/$distributionUrlName" + cat >"$javaSource" <<-END + public class Downloader extends java.net.Authenticator + { + protected java.net.PasswordAuthentication getPasswordAuthentication() + { + return new java.net.PasswordAuthentication( System.getenv( "MVNW_USERNAME" ), System.getenv( "MVNW_PASSWORD" ).toCharArray() ); + } + public static void main( String[] args ) throws Exception + { + setDefault( new Downloader() ); + java.nio.file.Files.copy( java.net.URI.create( args[0] ).toURL().openStream(), java.nio.file.Paths.get( args[1] ).toAbsolutePath().normalize() ); + } + } + END + # For Cygwin/MinGW, switch paths to Windows format before running javac and java + verbose " - Compiling Downloader.java ..." + "$(native_path "$JAVACCMD")" "$(native_path "$javaSource")" || die "Failed to compile Downloader.java" + verbose " - Running Downloader.java ..." + "$(native_path "$JAVACMD")" -cp "$(native_path "$TMP_DOWNLOAD_DIR")" Downloader "$distributionUrl" "$(native_path "$targetZip")" +fi + +# If specified, validate the SHA-256 sum of the Maven distribution zip file +if [ -n "${distributionSha256Sum-}" ]; then + distributionSha256Result=false + if [ "$MVN_CMD" = mvnd.sh ]; then + echo "Checksum validation is not supported for maven-mvnd." >&2 + echo "Please disable validation by removing 'distributionSha256Sum' from your maven-wrapper.properties." >&2 + exit 1 + elif command -v sha256sum >/dev/null; then + if echo "$distributionSha256Sum $TMP_DOWNLOAD_DIR/$distributionUrlName" | sha256sum -c >/dev/null 2>&1; then + distributionSha256Result=true + fi + elif command -v shasum >/dev/null; then + if echo "$distributionSha256Sum $TMP_DOWNLOAD_DIR/$distributionUrlName" | shasum -a 256 -c >/dev/null 2>&1; then + distributionSha256Result=true + fi + else + echo "Checksum validation was requested but neither 'sha256sum' or 'shasum' are available." >&2 + echo "Please install either command, or disable validation by removing 'distributionSha256Sum' from your maven-wrapper.properties." >&2 + exit 1 + fi + if [ $distributionSha256Result = false ]; then + echo "Error: Failed to validate Maven distribution SHA-256, your Maven distribution might be compromised." >&2 + echo "If you updated your Maven version, you need to update the specified distributionSha256Sum property." >&2 + exit 1 + fi +fi + +# unzip and move +if command -v unzip >/dev/null; then + unzip ${__MVNW_QUIET_UNZIP:+"$__MVNW_QUIET_UNZIP"} "$TMP_DOWNLOAD_DIR/$distributionUrlName" -d "$TMP_DOWNLOAD_DIR" || die "failed to unzip" +else + tar xzf${__MVNW_QUIET_TAR:+"$__MVNW_QUIET_TAR"} "$TMP_DOWNLOAD_DIR/$distributionUrlName" -C "$TMP_DOWNLOAD_DIR" || die "failed to untar" +fi +printf %s\\n "$distributionUrl" >"$TMP_DOWNLOAD_DIR/$distributionUrlNameMain/mvnw.url" +mv -- "$TMP_DOWNLOAD_DIR/$distributionUrlNameMain" "$MAVEN_HOME" || [ -d "$MAVEN_HOME" ] || die "fail to move MAVEN_HOME" + +clean || : +exec_maven "$@" diff --git a/mvnw.cmd b/mvnw.cmd @@ -0,0 +1,149 @@ +<# : batch portion +@REM ---------------------------------------------------------------------------- +@REM Licensed to the Apache Software Foundation (ASF) under one +@REM or more contributor license agreements. See the NOTICE file +@REM distributed with this work for additional information +@REM regarding copyright ownership. The ASF licenses this file +@REM to you under the Apache License, Version 2.0 (the +@REM "License"); you may not use this file except in compliance +@REM with the License. You may obtain a copy of the License at +@REM +@REM http://www.apache.org/licenses/LICENSE-2.0 +@REM +@REM Unless required by applicable law or agreed to in writing, +@REM software distributed under the License is distributed on an +@REM "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +@REM KIND, either express or implied. See the License for the +@REM specific language governing permissions and limitations +@REM under the License. +@REM ---------------------------------------------------------------------------- + +@REM ---------------------------------------------------------------------------- +@REM Apache Maven Wrapper startup batch script, version 3.3.2 +@REM +@REM Optional ENV vars +@REM MVNW_REPOURL - repo url base for downloading maven distribution +@REM MVNW_USERNAME/MVNW_PASSWORD - user and password for downloading maven +@REM MVNW_VERBOSE - true: enable verbose log; others: silence the output +@REM ---------------------------------------------------------------------------- + +@IF "%__MVNW_ARG0_NAME__%"=="" (SET __MVNW_ARG0_NAME__=%~nx0) +@SET __MVNW_CMD__= +@SET __MVNW_ERROR__= +@SET __MVNW_PSMODULEP_SAVE=%PSModulePath% +@SET PSModulePath= +@FOR /F "usebackq tokens=1* delims==" %%A IN (`powershell -noprofile "& {$scriptDir='%~dp0'; $script='%__MVNW_ARG0_NAME__%'; icm -ScriptBlock ([Scriptblock]::Create((Get-Content -Raw '%~f0'))) -NoNewScope}"`) DO @( + IF "%%A"=="MVN_CMD" (set __MVNW_CMD__=%%B) ELSE IF "%%B"=="" (echo %%A) ELSE (echo %%A=%%B) +) +@SET PSModulePath=%__MVNW_PSMODULEP_SAVE% +@SET __MVNW_PSMODULEP_SAVE= +@SET __MVNW_ARG0_NAME__= +@SET MVNW_USERNAME= +@SET MVNW_PASSWORD= +@IF NOT "%__MVNW_CMD__%"=="" (%__MVNW_CMD__% %*) +@echo Cannot start maven from wrapper >&2 && exit /b 1 +@GOTO :EOF +: end batch / begin powershell #> + +$ErrorActionPreference = "Stop" +if ($env:MVNW_VERBOSE -eq "true") { + $VerbosePreference = "Continue" +} + +# calculate distributionUrl, requires .mvn/wrapper/maven-wrapper.properties +$distributionUrl = (Get-Content -Raw "$scriptDir/.mvn/wrapper/maven-wrapper.properties" | ConvertFrom-StringData).distributionUrl +if (!$distributionUrl) { + Write-Error "cannot read distributionUrl property in $scriptDir/.mvn/wrapper/maven-wrapper.properties" +} + +switch -wildcard -casesensitive ( $($distributionUrl -replace '^.*/','') ) { + "maven-mvnd-*" { + $USE_MVND = $true + $distributionUrl = $distributionUrl -replace '-bin\.[^.]*$',"-windows-amd64.zip" + $MVN_CMD = "mvnd.cmd" + break + } + default { + $USE_MVND = $false + $MVN_CMD = $script -replace '^mvnw','mvn' + break + } +} + +# apply MVNW_REPOURL and calculate MAVEN_HOME +# maven home pattern: ~/.m2/wrapper/dists/{apache-maven-<version>,maven-mvnd-<version>-<platform>}/<hash> +if ($env:MVNW_REPOURL) { + $MVNW_REPO_PATTERN = if ($USE_MVND) { "/org/apache/maven/" } else { "/maven/mvnd/" } + $distributionUrl = "$env:MVNW_REPOURL$MVNW_REPO_PATTERN$($distributionUrl -replace '^.*'+$MVNW_REPO_PATTERN,'')" +} +$distributionUrlName = $distributionUrl -replace '^.*/','' +$distributionUrlNameMain = $distributionUrlName -replace '\.[^.]*$','' -replace '-bin$','' +$MAVEN_HOME_PARENT = "$HOME/.m2/wrapper/dists/$distributionUrlNameMain" +if ($env:MAVEN_USER_HOME) { + $MAVEN_HOME_PARENT = "$env:MAVEN_USER_HOME/wrapper/dists/$distributionUrlNameMain" +} +$MAVEN_HOME_NAME = ([System.Security.Cryptography.MD5]::Create().ComputeHash([byte[]][char[]]$distributionUrl) | ForEach-Object {$_.ToString("x2")}) -join '' +$MAVEN_HOME = "$MAVEN_HOME_PARENT/$MAVEN_HOME_NAME" + +if (Test-Path -Path "$MAVEN_HOME" -PathType Container) { + Write-Verbose "found existing MAVEN_HOME at $MAVEN_HOME" + Write-Output "MVN_CMD=$MAVEN_HOME/bin/$MVN_CMD" + exit $? +} + +if (! $distributionUrlNameMain -or ($distributionUrlName -eq $distributionUrlNameMain)) { + Write-Error "distributionUrl is not valid, must end with *-bin.zip, but found $distributionUrl" +} + +# prepare tmp dir +$TMP_DOWNLOAD_DIR_HOLDER = New-TemporaryFile +$TMP_DOWNLOAD_DIR = New-Item -Itemtype Directory -Path "$TMP_DOWNLOAD_DIR_HOLDER.dir" +$TMP_DOWNLOAD_DIR_HOLDER.Delete() | Out-Null +trap { + if ($TMP_DOWNLOAD_DIR.Exists) { + try { Remove-Item $TMP_DOWNLOAD_DIR -Recurse -Force | Out-Null } + catch { Write-Warning "Cannot remove $TMP_DOWNLOAD_DIR" } + } +} + +New-Item -Itemtype Directory -Path "$MAVEN_HOME_PARENT" -Force | Out-Null + +# Download and Install Apache Maven +Write-Verbose "Couldn't find MAVEN_HOME, downloading and installing it ..." +Write-Verbose "Downloading from: $distributionUrl" +Write-Verbose "Downloading to: $TMP_DOWNLOAD_DIR/$distributionUrlName" + +$webclient = New-Object System.Net.WebClient +if ($env:MVNW_USERNAME -and $env:MVNW_PASSWORD) { + $webclient.Credentials = New-Object System.Net.NetworkCredential($env:MVNW_USERNAME, $env:MVNW_PASSWORD) +} +[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 +$webclient.DownloadFile($distributionUrl, "$TMP_DOWNLOAD_DIR/$distributionUrlName") | Out-Null + +# If specified, validate the SHA-256 sum of the Maven distribution zip file +$distributionSha256Sum = (Get-Content -Raw "$scriptDir/.mvn/wrapper/maven-wrapper.properties" | ConvertFrom-StringData).distributionSha256Sum +if ($distributionSha256Sum) { + if ($USE_MVND) { + Write-Error "Checksum validation is not supported for maven-mvnd. `nPlease disable validation by removing 'distributionSha256Sum' from your maven-wrapper.properties." + } + Import-Module $PSHOME\Modules\Microsoft.PowerShell.Utility -Function Get-FileHash + if ((Get-FileHash "$TMP_DOWNLOAD_DIR/$distributionUrlName" -Algorithm SHA256).Hash.ToLower() -ne $distributionSha256Sum) { + Write-Error "Error: Failed to validate Maven distribution SHA-256, your Maven distribution might be compromised. If you updated your Maven version, you need to update the specified distributionSha256Sum property." + } +} + +# unzip and move +Expand-Archive "$TMP_DOWNLOAD_DIR/$distributionUrlName" -DestinationPath "$TMP_DOWNLOAD_DIR" | Out-Null +Rename-Item -Path "$TMP_DOWNLOAD_DIR/$distributionUrlNameMain" -NewName $MAVEN_HOME_NAME | Out-Null +try { + Move-Item -Path "$TMP_DOWNLOAD_DIR/$MAVEN_HOME_NAME" -Destination $MAVEN_HOME_PARENT | Out-Null +} catch { + if (! (Test-Path -Path "$MAVEN_HOME" -PathType Container)) { + Write-Error "fail to move MAVEN_HOME" + } +} finally { + try { Remove-Item $TMP_DOWNLOAD_DIR -Recurse -Force | Out-Null } + catch { Write-Warning "Cannot remove $TMP_DOWNLOAD_DIR" } +} + +Write-Output "MVN_CMD=$MAVEN_HOME/bin/$MVN_CMD" diff --git a/pom.xml b/pom.xml @@ -0,0 +1,68 @@ +<?xml version="1.0" encoding="UTF-8"?> +<project xmlns="http://maven.apache.org/POM/4.0.0" + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd"> + <modelVersion>4.0.0</modelVersion> + <parent> + <groupId>org.springframework.boot</groupId> + <artifactId>spring-boot-starter-parent</artifactId> + <version>3.4.3</version> + <relativePath /> <!-- lookup parent from repository --> + </parent> + + <groupId>fr.kevincorvisier.mtg</groupId> + <artifactId>lets-trade</artifactId> + <version>0.0.1-SNAPSHOT</version> + + <dependencies> + <dependency> + <groupId>org.springframework.boot</groupId> + <artifactId>spring-boot-starter-data-jpa</artifactId> + </dependency> + <dependency> + <groupId>org.springframework.boot</groupId> + <artifactId>spring-boot-starter-validation</artifactId> + </dependency> + <dependency> + <groupId>org.springframework.boot</groupId> + <artifactId>spring-boot-starter-web</artifactId> + </dependency> + + <!-- SQLite database --> + <dependency> + <groupId>org.flywaydb</groupId> + <artifactId>flyway-core</artifactId> + </dependency> + <dependency> + <groupId>org.hibernate.orm</groupId> + <artifactId>hibernate-community-dialects</artifactId> + </dependency> + <dependency> + <groupId>org.xerial</groupId> + <artifactId>sqlite-jdbc</artifactId> + </dependency> + + <dependency> + <groupId>org.projectlombok</groupId> + <artifactId>lombok</artifactId> + <optional>true</optional> + </dependency> + </dependencies> + + <build> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-compiler-plugin</artifactId> + <configuration> + <annotationProcessorPaths> + <path> + <groupId>org.projectlombok</groupId> + <artifactId>lombok</artifactId> + </path> + </annotationProcessorPaths> + </configuration> + </plugin> + </plugins> + </build> +</project> diff --git a/src/main/java/fr/kevincorvisier/mtg/letstrade/LetsTradeApplication.java b/src/main/java/fr/kevincorvisier/mtg/letstrade/LetsTradeApplication.java @@ -0,0 +1,15 @@ +package fr.kevincorvisier.mtg.letstrade; + +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.scheduling.annotation.EnableScheduling; + +@SpringBootApplication +@EnableScheduling +public class LetsTradeApplication +{ + public static void main(final String[] args) + { + SpringApplication.run(LetsTradeApplication.class, args); + } +} diff --git a/src/main/java/fr/kevincorvisier/mtg/letstrade/database/DatabaseConfiguration.java b/src/main/java/fr/kevincorvisier/mtg/letstrade/database/DatabaseConfiguration.java @@ -0,0 +1,20 @@ +package fr.kevincorvisier.mtg.letstrade.database; + +import javax.sql.DataSource; + +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.jdbc.datasource.DriverManagerDataSource; + +@Configuration +public class DatabaseConfiguration +{ + @Bean + public DataSource dataSource() + { + final DriverManagerDataSource dataSource = new DriverManagerDataSource(); + dataSource.setDriverClassName("org.sqlite.JDBC"); + dataSource.setUrl("jdbc:sqlite:your.db"); + return dataSource; + } +} diff --git a/src/main/java/fr/kevincorvisier/mtg/letstrade/database/entities/DBBinder.java b/src/main/java/fr/kevincorvisier/mtg/letstrade/database/entities/DBBinder.java @@ -0,0 +1,43 @@ +package fr.kevincorvisier.mtg.letstrade.database.entities; + +import java.util.Collection; + +import jakarta.persistence.Column; +import jakarta.persistence.GeneratedValue; +import jakarta.persistence.GenerationType; +import jakarta.persistence.Id; +import jakarta.persistence.ManyToOne; +import jakarta.persistence.OneToMany; +import jakarta.persistence.Version; +import jakarta.validation.constraints.NotNull; +import lombok.AccessLevel; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.NonNull; + +//@Entity(name = "binder") + +@Data +@NoArgsConstructor(access = AccessLevel.PRIVATE) +@AllArgsConstructor(access = AccessLevel.PRIVATE) +@Builder(toBuilder = true) +public class DBBinder +{ + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + @Column(updatable = false, nullable = false) + private Long id; + + @Version + private short version; + + @NonNull + @NotNull + @ManyToOne(optional = false) + private DBUser owner; + + @OneToMany + private Collection<DBBinderCard> cards; +} diff --git a/src/main/java/fr/kevincorvisier/mtg/letstrade/database/entities/DBBinderCard.java b/src/main/java/fr/kevincorvisier/mtg/letstrade/database/entities/DBBinderCard.java @@ -0,0 +1,28 @@ +package fr.kevincorvisier.mtg.letstrade.database.entities; + +import jakarta.persistence.ManyToMany; +import lombok.AccessLevel; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +//@Entity(name = "binder_card") + +/** + * Represents one or several identical printed card(s) in a user binder + */ +@Data +@NoArgsConstructor(access = AccessLevel.PRIVATE) +@AllArgsConstructor(access = AccessLevel.PRIVATE) +@Builder(toBuilder = true) +public class DBBinderCard +{ + @ManyToMany + private DBBinder binder; + + @ManyToMany + private DBPrintedCard card; + + private int quantity; +} diff --git a/src/main/java/fr/kevincorvisier/mtg/letstrade/database/entities/DBOracleCard.java b/src/main/java/fr/kevincorvisier/mtg/letstrade/database/entities/DBOracleCard.java @@ -0,0 +1,53 @@ +package fr.kevincorvisier.mtg.letstrade.database.entities; + +import java.util.UUID; + +import org.hibernate.annotations.NaturalId; + +import jakarta.persistence.Column; +import jakarta.persistence.Entity; +import jakarta.persistence.GeneratedValue; +import jakarta.persistence.GenerationType; +import jakarta.persistence.Id; +import jakarta.persistence.Version; +import jakarta.validation.constraints.NotNull; +import lombok.AccessLevel; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.NonNull; + +@Entity(name = "oracle_card") + +@Data +@NoArgsConstructor(access = AccessLevel.PRIVATE) +@AllArgsConstructor(access = AccessLevel.PRIVATE) +@Builder(toBuilder = true) +public class DBOracleCard +{ + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + @Column(updatable = false, nullable = false) + private Long id; + + @Version + private short version; + + /** + * A unique ID for this card in Scryfall’s database. + */ + @NonNull + @NotNull + @NaturalId + @Column(updatable = false, nullable = false, unique = true) + private UUID scryfallId; + + /** + * The name of this card. If this card has multiple faces, this field will contain both names separated by ␣//␣. + */ + @NonNull + @NotNull + @Column(nullable = false) + private String name; +} diff --git a/src/main/java/fr/kevincorvisier/mtg/letstrade/database/entities/DBPrintedCard.java b/src/main/java/fr/kevincorvisier/mtg/letstrade/database/entities/DBPrintedCard.java @@ -0,0 +1,72 @@ +package fr.kevincorvisier.mtg.letstrade.database.entities; + +import java.util.UUID; + +import org.hibernate.annotations.NaturalId; + +import jakarta.persistence.Column; +import jakarta.persistence.Entity; +import jakarta.persistence.GeneratedValue; +import jakarta.persistence.GenerationType; +import jakarta.persistence.Id; +import jakarta.persistence.ManyToOne; +import jakarta.persistence.Version; +import jakarta.validation.constraints.NotNull; +import lombok.AccessLevel; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.NonNull; + +@Entity(name = "card") + +@Data +@NoArgsConstructor(access = AccessLevel.PRIVATE) +@AllArgsConstructor(access = AccessLevel.PRIVATE) +@Builder(toBuilder = true) +public class DBPrintedCard +{ + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + @Column(updatable = false, nullable = false) + private Long id; + + @Version + private short version; + + /** + * A unique ID for this card in Scryfall’s database. + */ + @NonNull + @NotNull + @NaturalId + @Column(updatable = false, nullable = false, unique = true) + private UUID scryfallId; + + @NonNull + @NotNull + @ManyToOne(optional = false) + private DBOracleCard oracleCard; + + @NonNull + @NotNull + @ManyToOne(optional = false) + private DBSet set; + + /** + * A language code for this printing. + */ + @NonNull + @NotNull + @Column(nullable = false) + private String lang; + + /** + * This card’s collector number. Note that collector numbers can contain non-numeric characters, such as letters or ★. + */ + @NonNull + @NotNull + @Column(nullable = false) + private String collectorNumber; +} diff --git a/src/main/java/fr/kevincorvisier/mtg/letstrade/database/entities/DBSet.java b/src/main/java/fr/kevincorvisier/mtg/letstrade/database/entities/DBSet.java @@ -0,0 +1,61 @@ +package fr.kevincorvisier.mtg.letstrade.database.entities; + +import java.util.UUID; + +import org.hibernate.annotations.NaturalId; + +import jakarta.persistence.Column; +import jakarta.persistence.Entity; +import jakarta.persistence.GeneratedValue; +import jakarta.persistence.GenerationType; +import jakarta.persistence.Id; +import jakarta.persistence.Version; +import jakarta.validation.constraints.NotNull; +import lombok.AccessLevel; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.NonNull; + +@Entity(name = "mtgset") + +@Data +@NoArgsConstructor(access = AccessLevel.PRIVATE) +@AllArgsConstructor(access = AccessLevel.PRIVATE) +@Builder(toBuilder = true) +public class DBSet +{ + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + @Column(updatable = false, nullable = false) + private Long id; + + @Version + private short version; + + /** + * A unique ID for this set on Scryfall that will not change. + */ + @NonNull + @NotNull + @NaturalId + @Column(updatable = false, nullable = false, unique = true) + private UUID scryfallId; + + /** + * The unique three to six-letter code for this set. + */ + @NonNull + @NotNull + @Column(nullable = false, unique = true) + private String code; + + /** + * The English name of the set. + */ + @NonNull + @NotNull + @Column(nullable = false, unique = true) + private String name; +} diff --git a/src/main/java/fr/kevincorvisier/mtg/letstrade/database/entities/DBUser.java b/src/main/java/fr/kevincorvisier/mtg/letstrade/database/entities/DBUser.java @@ -0,0 +1,5 @@ +package fr.kevincorvisier.mtg.letstrade.database.entities; + +public class DBUser +{ +} diff --git a/src/main/java/fr/kevincorvisier/mtg/letstrade/database/repositories/DBCardRepository.java b/src/main/java/fr/kevincorvisier/mtg/letstrade/database/repositories/DBCardRepository.java @@ -0,0 +1,15 @@ +package fr.kevincorvisier.mtg.letstrade.database.repositories; + +import java.util.Optional; +import java.util.UUID; + +import org.springframework.data.repository.CrudRepository; + +import fr.kevincorvisier.mtg.letstrade.database.entities.DBPrintedCard; + +public interface DBCardRepository extends CrudRepository<DBPrintedCard, Long> +{ + Iterable<DBPrintedCard> findAllByScryfallIdIn(final Iterable<UUID> ids); + + Optional<DBPrintedCard> findByScryfallId(final UUID id); +} diff --git a/src/main/java/fr/kevincorvisier/mtg/letstrade/database/repositories/DBOracleCardRepository.java b/src/main/java/fr/kevincorvisier/mtg/letstrade/database/repositories/DBOracleCardRepository.java @@ -0,0 +1,15 @@ +package fr.kevincorvisier.mtg.letstrade.database.repositories; + +import java.util.Optional; +import java.util.UUID; + +import org.springframework.data.repository.CrudRepository; + +import fr.kevincorvisier.mtg.letstrade.database.entities.DBOracleCard; + +public interface DBOracleCardRepository extends CrudRepository<DBOracleCard, Long> +{ + Iterable<DBOracleCard> findAllByScryfallIdIn(final Iterable<UUID> ids); + + Optional<DBOracleCard> findByScryfallId(final UUID id); +} diff --git a/src/main/java/fr/kevincorvisier/mtg/letstrade/database/repositories/DBSetRepository.java b/src/main/java/fr/kevincorvisier/mtg/letstrade/database/repositories/DBSetRepository.java @@ -0,0 +1,15 @@ +package fr.kevincorvisier.mtg.letstrade.database.repositories; + +import java.util.Optional; +import java.util.UUID; + +import org.springframework.data.repository.CrudRepository; + +import fr.kevincorvisier.mtg.letstrade.database.entities.DBSet; + +public interface DBSetRepository extends CrudRepository<DBSet, Long> +{ + Iterable<DBSet> findAllByScryfallIdIn(final Iterable<UUID> ids); + + Optional<DBSet> findByScryfallId(final UUID id); +} diff --git a/src/main/java/fr/kevincorvisier/mtg/letstrade/scryfall/ScryfallRestClient.java b/src/main/java/fr/kevincorvisier/mtg/letstrade/scryfall/ScryfallRestClient.java @@ -0,0 +1,159 @@ +package fr.kevincorvisier.mtg.letstrade.scryfall; + +import java.io.File; +import java.io.IOException; +import java.util.Collection; +import java.util.HashSet; +import java.util.Map; +import java.util.UUID; +import java.util.function.Consumer; +import java.util.function.Predicate; + +import org.springframework.boot.web.client.RestTemplateBuilder; +import org.springframework.stereotype.Service; +import org.springframework.web.client.RestTemplate; + +import com.fasterxml.jackson.core.JsonFactory; +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.core.JsonToken; + +import fr.kevincorvisier.mtg.letstrade.scryfall.domain.ScryfallBulkData; +import fr.kevincorvisier.mtg.letstrade.scryfall.domain.ScryfallCard; +import fr.kevincorvisier.mtg.letstrade.scryfall.domain.ScryfallCard.ScryfallCardBuilder; +import jakarta.validation.constraints.NotNull; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +@Service +@RequiredArgsConstructor +public class ScryfallRestClient +{ + private final RestTemplate restTemplate = new RestTemplateBuilder().rootUri("https://api.scryfall.com").build(); + + public void getCards(final Predicate<ScryfallCard> filter, final Consumer<Iterable<ScryfallCard>> consumer, final int batchSize) throws IOException + { + // final ScryfallBulkData bulkData = getBulkDataByType("oracle_cards"); + + final JsonFactory jfactory = new JsonFactory(); + // final JsonParser jParser = jfactory.createParser(bulkData.getDownloadUri().toURL()); + final JsonParser jParser = jfactory.createParser(new File("/home/kebi/tmp/all-cards-20250315092148.json")); + // final JsonParser jParser = jfactory.createParser(new File("/home/kebi/tmp/default-cards-20250313214259.json")); + + long nb = 0; + Collection<ScryfallCard> batch = new HashSet<>(); + + if (jParser.nextToken() == JsonToken.START_ARRAY) + { + while (jParser.nextToken() != JsonToken.END_ARRAY) + { + if (jParser.currentToken() == JsonToken.START_OBJECT) + { + final ScryfallCard card = readCard(jParser); + if (!filter.test(card)) + continue; + + batch.add(card); + nb++; + + if (batch.size() == batchSize) + { + consumer.accept(batch); + batch = new HashSet<>(); + } + } + } + } + + log.info("getCards {} cards read", nb); + } + + @NotNull + private ScryfallCard readCard(final JsonParser jParser) throws IOException + { + final ScryfallCardBuilder builder = ScryfallCard.builder(); + + try + { + while (jParser.nextToken() != JsonToken.END_OBJECT) + { + switch (jParser.currentName()) + { + case "id": + jParser.nextToken(); + builder.id(UUID.fromString(jParser.getText())); + break; + case "oracle_id": + jParser.nextToken(); + builder.oracleId(UUID.fromString(jParser.getText())); + break; + case "name": + jParser.nextToken(); + builder.name(jParser.getText()); + break; + case "lang": + jParser.nextToken(); + builder.lang(jParser.getText()); + break; + case "set": + jParser.nextToken(); + builder.set(jParser.getText()); + break; + case "set_name": + jParser.nextToken(); + builder.setName(jParser.getText()); + break; + case "set_id": + jParser.nextToken(); + builder.setId(UUID.fromString(jParser.getText())); + break; + case "collector_number": + jParser.nextToken(); + builder.collectorNumber(jParser.getText()); + break; + case "games": + builder.games(readArray(jParser)); + break; + case "promo_types": + builder.promoTypes(readArray(jParser)); + break; + default: + jParser.nextToken(); + jParser.skipChildren(); + break; + } + } + + return builder.build(); + } + catch (final Exception e) + { + log.error("Error while decoding {}", builder, e); + throw e; + } + + } + + @NotNull + private Collection<String> readArray(@NotNull final JsonParser jParser) throws IOException + { + final Collection<String> values = new HashSet<>(); + + if (jParser.nextToken() == JsonToken.START_ARRAY) + { + while (jParser.nextToken() != JsonToken.END_ARRAY) + { + values.add(jParser.getText()); + } + } + + return values; + } + + public ScryfallBulkData getBulkDataByType(@NotNull final String type) + { + final Map<String, String> uriVariables = Map.of("type", type); + + return restTemplate.getForObject("/bulk-data/{type}", ScryfallBulkData.class, uriVariables); + } +} diff --git a/src/main/java/fr/kevincorvisier/mtg/letstrade/scryfall/domain/ScryfallBulkData.java b/src/main/java/fr/kevincorvisier/mtg/letstrade/scryfall/domain/ScryfallBulkData.java @@ -0,0 +1,14 @@ +package fr.kevincorvisier.mtg.letstrade.scryfall.domain; + +import java.net.URI; + +import com.fasterxml.jackson.annotation.JsonProperty; + +import lombok.Data; + +@Data +public class ScryfallBulkData +{ + @JsonProperty("download_uri") + private URI downloadUri; +} diff --git a/src/main/java/fr/kevincorvisier/mtg/letstrade/scryfall/domain/ScryfallCard.java b/src/main/java/fr/kevincorvisier/mtg/letstrade/scryfall/domain/ScryfallCard.java @@ -0,0 +1,80 @@ +package fr.kevincorvisier.mtg.letstrade.scryfall.domain; + +import java.util.Collection; +import java.util.UUID; + +import jakarta.annotation.Nullable; +import jakarta.validation.constraints.NotNull; +import lombok.Builder; +import lombok.Data; +import lombok.NonNull; + +@Data +@Builder +public class ScryfallCard +{ + /** + * A unique ID for this card in Scryfall’s database. + */ + @NonNull + @NotNull + private UUID id; + + @Nullable + private UUID oracleId; + + /** + * The name of this card. If this card has multiple faces, this field will contain both names separated by ␣//␣. + */ + @NonNull + @NotNull + private String name; + + /** + * A language code for this printing. + */ + @NonNull + @NotNull + private String lang; + + /** + * A list of games that this card print is available in, paper, arena, and/or mtgo. + */ + @NonNull + @NotNull + private Collection<String> games; + + /** + * This card’s set code. + */ + @NonNull + @NotNull + private String set; + + /** + * This card’s full set name. + */ + @NonNull + @NotNull + private String setName; + + /** + * This card’s Set object UUID. + */ + @NonNull + @NotNull + private UUID setId; + + /** + * This card’s collector number. Note that collector numbers can contain non-numeric characters, such as letters or ★. + */ + @NonNull + @NotNull + private String collectorNumber; + + /** + * An array of strings describing what categories of promo cards this card falls into. + */ + @Nullable + private Collection<String> promoTypes; +} diff --git a/src/main/java/fr/kevincorvisier/mtg/letstrade/tasks/ScryfallSynchronizationTask.java b/src/main/java/fr/kevincorvisier/mtg/letstrade/tasks/ScryfallSynchronizationTask.java @@ -0,0 +1,223 @@ +package fr.kevincorvisier.mtg.letstrade.tasks; + +import java.io.IOException; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; + +import org.springframework.beans.factory.annotation.Value; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.scheduling.concurrent.CustomizableThreadFactory; +import org.springframework.stereotype.Component; + +import fr.kevincorvisier.mtg.letstrade.database.entities.DBOracleCard; +import fr.kevincorvisier.mtg.letstrade.database.entities.DBOracleCard.DBOracleCardBuilder; +import fr.kevincorvisier.mtg.letstrade.database.entities.DBPrintedCard; +import fr.kevincorvisier.mtg.letstrade.database.entities.DBPrintedCard.DBPrintedCardBuilder; +import fr.kevincorvisier.mtg.letstrade.database.entities.DBSet; +import fr.kevincorvisier.mtg.letstrade.database.entities.DBSet.DBSetBuilder; +import fr.kevincorvisier.mtg.letstrade.database.repositories.DBCardRepository; +import fr.kevincorvisier.mtg.letstrade.database.repositories.DBOracleCardRepository; +import fr.kevincorvisier.mtg.letstrade.database.repositories.DBSetRepository; +import fr.kevincorvisier.mtg.letstrade.scryfall.ScryfallRestClient; +import fr.kevincorvisier.mtg.letstrade.scryfall.domain.ScryfallCard; +import jakarta.validation.constraints.NotNull; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +@Component +@RequiredArgsConstructor +public class ScryfallSynchronizationTask +{ + private final ExecutorService executorService = Executors.newSingleThreadExecutor(new CustomizableThreadFactory("scryfall-synchronization-thread-")); + private final DBOracleCardRepository oracleCardRepository; + private final DBSetRepository setRepository; + private final DBCardRepository cardRepository; + + @Value("${tasks.scryfall-synchronization.enabled}") + private final boolean enabled; + @Value("${spring.jpa.properties.hibernate.jdbc.batch_size}") + private final int batchSize; + + private final ScryfallRestClient scryfall; + + @Scheduled(fixedRate = 1, timeUnit = TimeUnit.DAYS) + public void synchronize() + { + try + { + doSynchronize(); + } + catch (final Exception e) + { + log.error("synchronize", e); + } + } + + private void doSynchronize() throws IOException + { + if (!enabled) + { + log.info("doSynchronize: synchronization disabled"); + return; + } + + final long start = System.currentTimeMillis(); + + scryfall.getCards(this::accept, this::saveCards, batchSize); + log.info("Synchronization took {} ms", System.currentTimeMillis() - start); + } + + private boolean accept(final ScryfallCard card) + { + // Ignore non-paper cards + if (!card.getGames().contains("paper")) + return false; + + // Ignore cards printed on thick cardstock + final Collection<String> promoTypes = card.getPromoTypes(); + if (promoTypes != null && promoTypes.contains("thick")) + return false; + + final UUID oracleId = card.getOracleId(); + if (oracleId == null) + return false; + + return true; + } + + private void saveCards(final Iterable<ScryfallCard> cards) + { + executorService.execute(() -> { + final Map<UUID, DBOracleCard> oracleCards = saveOracleCards(cards); + final Map<UUID, DBSet> sets = saveSets(cards); + savePrintedCards(cards, oracleCards, sets); + }); + } + + @NotNull + private Map<UUID, DBOracleCard> saveOracleCards(final Iterable<ScryfallCard> cards) + { + final Collection<UUID> scryfallIds = new HashSet<>(); + for (final ScryfallCard card : cards) + scryfallIds.add(card.getOracleId()); + + final Map<UUID, DBOracleCard> existingByScryfallId = new HashMap<>(); + for (final DBOracleCard oracleCard : oracleCardRepository.findAllByScryfallIdIn(scryfallIds)) + existingByScryfallId.put(oracleCard.getScryfallId(), oracleCard); + + final Collection<DBOracleCard> toSave = new HashSet<>(); + + for (final ScryfallCard card : cards) + { + final UUID oracleId = card.getOracleId(); + if (oracleId == null) + continue; + + final DBOracleCard existing = existingByScryfallId.get(oracleId); + final DBOracleCardBuilder builder = existing != null ? existing.toBuilder() : DBOracleCard.builder().scryfallId(oracleId); + + builder.name(card.getName()); + final DBOracleCard updated = builder.build(); + + if (existing == null || !existing.equals(updated)) + toSave.add(updated); + } + + for (final DBOracleCard oracleCard : oracleCardRepository.saveAll(toSave)) + existingByScryfallId.put(oracleCard.getScryfallId(), oracleCard); + + return Collections.unmodifiableMap(existingByScryfallId); + } + + @NotNull + private Map<UUID, DBSet> saveSets(final Iterable<ScryfallCard> cards) + { + final Collection<UUID> scryfallIds = new HashSet<>(); + for (final ScryfallCard card : cards) + scryfallIds.add(card.getSetId()); + + final Map<UUID, DBSet> existingByScryfallId = new HashMap<>(); + for (final DBSet set : setRepository.findAllByScryfallIdIn(scryfallIds)) + existingByScryfallId.put(set.getScryfallId(), set); + + final Collection<DBSet> toSave = new HashSet<>(); + + for (final ScryfallCard card : cards) + { + final DBSet existing = existingByScryfallId.get(card.getSetId()); + final DBSetBuilder builder = existing != null ? existing.toBuilder() : DBSet.builder().scryfallId(card.getSetId()); + + builder.code(card.getSet()); + builder.name(card.getSetName()); + final DBSet updated = builder.build(); + + if (existing == null || !existing.equals(updated)) + toSave.add(updated); + } + + for (final DBSet set : setRepository.saveAll(toSave)) + existingByScryfallId.put(set.getScryfallId(), set); + + return Collections.unmodifiableMap(existingByScryfallId); + } + + private void savePrintedCards(final Iterable<ScryfallCard> cards, final Map<UUID, DBOracleCard> oracleCards, final Map<UUID, DBSet> sets) + { + final Collection<UUID> scryfallIds = new HashSet<>(); + for (final ScryfallCard card : cards) + scryfallIds.add(card.getId()); + + final Map<UUID, DBPrintedCard> existingByScryfallId = new HashMap<>(); + for (final DBPrintedCard card : cardRepository.findAllByScryfallIdIn(scryfallIds)) + existingByScryfallId.put(card.getScryfallId(), card); + + final Collection<DBPrintedCard> toSave = new HashSet<>(); + + for (final ScryfallCard card : cards) + { + final UUID oracleId = card.getOracleId(); + if (oracleId == null) + { + log.warn("Card without oracle_id: {}", card.getId()); + continue; + } + + final DBOracleCard oracleCard = oracleCards.get(oracleId); + if (oracleCard == null) + { + log.error("Unable to find oracle card for card: {}", card.getId()); + continue; + } + + final DBSet set = sets.get(card.getSetId()); + if (set == null) + { + log.error("Unable to find set for card: {}", card.getId()); + continue; + } + + final DBPrintedCard existing = existingByScryfallId.get(card.getId()); + final DBPrintedCardBuilder builder = existing != null ? existing.toBuilder() : DBPrintedCard.builder().scryfallId(card.getId()); + + builder.oracleCard(oracleCard); + builder.lang(card.getLang()); + builder.set(set); + builder.collectorNumber(card.getCollectorNumber()); + final DBPrintedCard updated = builder.build(); + + if (existing == null || !existing.equals(updated)) + toSave.add(updated); + } + + for (final DBPrintedCard card : cardRepository.saveAll(toSave)) + existingByScryfallId.put(card.getScryfallId(), card); + } +} diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties @@ -0,0 +1,11 @@ +tasks.scryfall-synchronization.enabled=true +spring.jpa.database-platform=org.hibernate.community.dialect.SQLiteDialect + +# spring.jpa.hibernate.ddl-auto=validate +spring.jpa.hibernate.ddl-auto=create +spring.flyway.enabled=false + +spring.jpa.properties.hibernate.jdbc.batch_size=10000 +spring.jpa.properties.hibernate.order_inserts=true +spring.jpa.properties.hibernate.order_updates=true +spring.jpa.properties.hibernate.generate_statistics=false +\ No newline at end of file diff --git a/src/main/resources/db/migration/V0_0_1.sql b/src/main/resources/db/migration/V0_0_1.sql @@ -0,0 +1,10 @@ +CREATE TABLE card ( + id bigint, + version smallint not null, + scryfall_id blob not null, + name varchar(255) not null, + lang varchar(255) not null, + set_code varchar(255) not null, + collector_number varchar(255) not null, + primary key (id) +); +\ No newline at end of file