SpringBatch
DB to DB 프로젝트 생성하기 (Spring Batch 5.0.x)
노민지
2024. 2. 5. 13:13
Spring Batch의 개념과 개발 환경 세팅
0. 개념 Batch Batch 작업이란, 데이터를 실시간으로 처리하는게 아니라, 일괄적으로 모아서 처리하는 작업을 의미한다. 하루동안 쌓인 데이터를 배치작업을 통해 특정 시간에 한 번에 처리하는 경
miji-it.tistory.com
Spring Batch project 만들기 전 DB 설정하기
1. DB 연결 기존 구축 되어 있는 MariaDB service type을 NodePort로 수정 apiVersion: v1 kind: Service ... 생략 ... type: NodePort status: loadBalancer: {} NodePort 를 확인한다. root@edu7:~# k get svc -n minji NAME TYPE CLUSTER-IP EXTERNAL
miji-it.tistory.com
위의 과정을 모두 진행했다는 전제로 진행하는 프로젝트이다.
뭔가 이상하게 안 된다 싶으면 application.yml에서 발생한 확률이 큰 듯하다.
버전이 안 맞는 다는 이유로 뭐가 참 안 된다.
resources/application.yml
spring:
profiles:
active: mariadb
---
spring:
profiles:
active: local
datasource:
hikari:
jdbc-url: jdbc:h2:mem:db;MODE=MYSQL;
username: sa
password:
driver-class-name: org.h2.Driver
batch:
job:
enabled: true
---
spring:
profiles:
active: mariadb
datasource:
hikari:
jdbc-url: jdbc:mariadb://{{ip}}:{{port}}/my_database
username: {{username}}
password: {{password}}
driver-class-name: org.mariadb.jdbc.Driver
jpa:
hibernate:
ddl-auto: update
naming:
physical-strategy: org.hibernate.boot.model.naming.PhysicalNamingStrategyStandardImpl
properties:
hibernate:
show_sql: true
format_sql: true
batch:
job:
enabled: true
name: ${job.name:NONE}
jdbc:
initialize-schema: always
build.gradle
plugins {
id 'java'
id 'org.springframework.boot' version '3.0.5'
id 'io.spring.dependency-management' version '1.1.0'
}
group = 'com.example'
version = '0.0.1-SNAPSHOT'
sourceCompatibility = '17'
configurations {
compileOnly {
extendsFrom annotationProcessor
}
}
repositories {
mavenCentral()
}
dependencies {
// Spring Batch
implementation 'org.springframework.boot:spring-boot-starter-batch'
// Spring Data JPA
implementation 'org.springframework.boot:spring-boot-starter-data-jpa'
// h2 DB
implementation 'com.h2database:h2'
// 스프링 웹
implementation 'org.springframework.boot:spring-boot-starter-web'
// 롬복
compileOnly 'org.projectlombok:lombok'
annotationProcessor 'org.projectlombok:lombok'
testImplementation 'org.springframework.boot:spring-boot-starter-test'
// mariadb
implementation 'org.mariadb.jdbc:mariadb-java-client:2.7.4'
testImplementation 'org.springframework.boot:spring-boot-starter-test'
}
tasks.named('test') {
useJUnitPlatform()
}
entity/SourcePerson.java
package com.example.playground.entity;
import jakarta.persistence.*;
import lombok.Getter;
import lombok.Setter;
import lombok.ToString;
import java.util.Date;
@Entity
@Getter
@Setter
@ToString
@Table(name = "SOURCE_PERSON")
public class SourcePerson {
@Id
@GeneratedValue(strategy = GenerationType.AUTO)
private int id;
private String lastName;
private String firstName;
private Date birthDate;
public SourcePerson() {
}
public SourcePerson(int id, String firstName, String lastName, Date birthDate) {
this.id = id;
this.firstName = firstName;
this.lastName = lastName;
this.birthDate = birthDate;
}
@Override
public String toString() {
return "id:"+ id+"firstName: " + firstName + ", lastName: " + lastName+ ", birthDate: " + birthDate;
}
}
entity/TargetPerson.java
package com.example.playground.entity;
import jakarta.persistence.*;
import lombok.Getter;
import lombok.Setter;
import java.util.Date;
@Entity
@Getter
@Setter
@Table(name = "TARGET_PERSON")
public class TargetPerson {
@Id
@GeneratedValue(strategy = GenerationType.AUTO)
private int id;
private String lastName;
private String firstName;
private Date birthDate;
public TargetPerson() {
}
public TargetPerson(int id,String firstName, String lastName, Date birthDate) {
this.id = id;
this.firstName = firstName;
this.lastName = lastName;
this.birthDate = birthDate;
}
@Override
public String toString() {
return "id: " + id + "firstName: " + firstName + ", lastName: " + lastName+ ", birthDate: " + birthDate;
}
}
repository/SourcePersonRepository.java
package com.example.playground.repository;
import com.example.playground.entity.SourcePerson;
import org.springframework.data.jpa.repository.JpaRepository;
public interface SourcePersonRepository extends JpaRepository<SourcePerson, Integer>{
}
repository/TargetPersonRepository.java
package com.example.playground.repository;
import com.example.playground.entity.TargetPerson;
import org.springframework.data.jpa.repository.JpaRepository;
public interface TargetPersonRepository extends JpaRepository<TargetPerson, Integer> {
}
PersonItemProcessor.java
package com.example.playground;
import com.example.playground.entity.SourcePerson;
import com.example.playground.entity.TargetPerson;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.batch.item.ItemProcessor;
import java.util.Date;
public class PersonItemProcessor implements ItemProcessor<SourcePerson, TargetPerson> {
private static final Logger log = LoggerFactory.getLogger(PersonItemProcessor.class);
@Override
public TargetPerson process(final SourcePerson sourcePerson) throws Exception {
final int id = sourcePerson.getId();
final String firstName = sourcePerson.getFirstName().toUpperCase();
final String lastName = sourcePerson.getLastName().toUpperCase();
final Date birthDate = sourcePerson.getBirthDate();
final TargetPerson transformedSourcePerson = new TargetPerson(firstName, lastName, birthDate);
log.info("Converting (" + sourcePerson + ") into (" + transformedSourcePerson + ")");
return transformedSourcePerson;
}
}
config/DBtoDBJobConfiguration.java
package com.example.playground.config;
import com.example.playground.PersonItemProcessor;
import com.example.playground.entity.SourcePerson;
import com.example.playground.entity.TargetPerson;
import com.example.playground.repository.SourcePersonRepository;
import com.example.playground.repository.TargetPersonRepository;
import lombok.extern.slf4j.Slf4j;
import org.springframework.batch.core.Job;
import org.springframework.batch.core.Step;
import org.springframework.batch.core.job.builder.JobBuilder;
import org.springframework.batch.core.launch.support.RunIdIncrementer;
import org.springframework.batch.core.repository.JobRepository;
import org.springframework.batch.core.step.builder.StepBuilder;
import org.springframework.batch.item.data.RepositoryItemReader;
import org.springframework.batch.item.data.RepositoryItemWriter;
import org.springframework.batch.item.data.builder.RepositoryItemReaderBuilder;
import org.springframework.batch.item.data.builder.RepositoryItemWriterBuilder;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.data.domain.Sort;
import org.springframework.transaction.PlatformTransactionManager;
import javax.sql.DataSource;
import java.util.Map;
@Slf4j
@Configuration
public class DBtoDBJobConfiguration {
@Autowired
private SourcePersonRepository sourcePersonRepo;
@Autowired
private TargetPersonRepository targetPersonRepo;
@Bean
public Job db2dbJob(JobRepository jobRepository, Step step1) {
return new JobBuilder("db2dbJob", jobRepository)
.incrementer(new RunIdIncrementer())
.flow(step1)
.end()
.build();
}
@Bean
public Step step1(JobRepository jobRepository, RepositoryItemReader<SourcePerson> reader, RepositoryItemWriter<TargetPerson> writer, PlatformTransactionManager platformTransactionManager){
return new StepBuilder("step1", jobRepository)
.<SourcePerson,TargetPerson>chunk(5, platformTransactionManager)
.reader(reader)
.processor(new PersonItemProcessor())
.writer(writer)
.build();
}
@Bean
public RepositoryItemReader<SourcePerson> repositoryItemReader() {
return new RepositoryItemReaderBuilder<SourcePerson>()
.name("repositoryItemReader")
.repository(sourcePersonRepo)
.methodName("findAll")
.pageSize(5)
.sorts(Map.of("id", Sort.Direction.ASC))
.build();
}
@Bean
public PersonItemProcessor processor() {
return new PersonItemProcessor();
}
@Bean
public RepositoryItemWriter<TargetPerson> writer(DataSource dataSource) {
return new RepositoryItemWriterBuilder<TargetPerson>()
.methodName("save")
.repository(targetPersonRepo)
.build();
}
}