diff --git a/exportAndStreambijiao/.gitignore b/exportAndStreambijiao/.gitignore new file mode 100644 index 0000000..682d4ed --- /dev/null +++ b/exportAndStreambijiao/.gitignore @@ -0,0 +1,27 @@ +# Created by .ignore support plugin (hsz.mobi) +### Java template +# Compiled class file +*.class + +# Log file +*.log + +# BlueJ files +*.ctxt + +# Mobile Tools for Java (J2ME) +.mtj.tmp/ + +# Package Files # +*.jar +*.war +*.nar +*.ear +*.zip +*.tar.gz +*.rar + +# virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml +hs_err_pid* +/.idea/ +/target/ diff --git a/exportAndStreambijiao/fanfu-web.iml b/exportAndStreambijiao/fanfu-web.iml new file mode 100644 index 0000000..4fb2f38 --- /dev/null +++ b/exportAndStreambijiao/fanfu-web.iml @@ -0,0 +1,92 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/exportAndStreambijiao/pom.xml b/exportAndStreambijiao/pom.xml new file mode 100644 index 0000000..7d184ca --- /dev/null +++ b/exportAndStreambijiao/pom.xml @@ -0,0 +1,68 @@ + + + 4.0.0 + + org.springframework.boot + spring-boot-starter-parent + 2.3.9.RELEASE + + + + com.ayi + woniu-web + 0.0.1-SNAPSHOT + woniou-web + create by woniu + + jar + + + 1.8 + + + + + mysql + mysql-connector-java + 8.0.29 + + + com.baomidou + mybatis-plus-boot-starter + 3.5.2 + + + org.springframework.boot + spring-boot-starter + + + org.springframework.boot + spring-boot-starter-web + + + org.springframework.boot + spring-boot-starter-test + test + + + org.projectlombok + lombok + + + org.assertj + assertj-core + + + + + + + + org.springframework.boot + spring-boot-maven-plugin + + + + + diff --git a/exportAndStreambijiao/src/main/java/com/woniu/WoNiuApplication.java b/exportAndStreambijiao/src/main/java/com/woniu/WoNiuApplication.java new file mode 100644 index 0000000..dc84452 --- /dev/null +++ b/exportAndStreambijiao/src/main/java/com/woniu/WoNiuApplication.java @@ -0,0 +1,22 @@ +package com.woniu; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.scheduling.annotation.EnableScheduling; + +@SpringBootApplication +@EnableScheduling +@Slf4j +public class WoNiuApplication { + + public static void main(String[] args) { + SpringApplication.run(WoNiuApplication.class, args); + log.info("test_info"); + log.error("test_error"); + log.warn("test_warn"); + } + + + +} diff --git a/exportAndStreambijiao/src/main/java/com/woniu/controller/HelloController.java b/exportAndStreambijiao/src/main/java/com/woniu/controller/HelloController.java new file mode 100644 index 0000000..34c5a46 --- /dev/null +++ b/exportAndStreambijiao/src/main/java/com/woniu/controller/HelloController.java @@ -0,0 +1,44 @@ +package com.woniu.controller; + +import com.woniu.service.AuthorsService; +import lombok.extern.slf4j.Slf4j; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; + +import javax.servlet.http.HttpServletResponse; +import java.io.IOException; + +/** + * SpringBoot 实现 MySQL 百万级数据量导出并避免 OOM 的解决方案 + * 比较stream流读取与传统方案对内存占用的大小 + */ +@RestController +@RequestMapping("download") +@Slf4j +public class HelloController { + private final AuthorsService authorsService; + + public HelloController(AuthorsService authorsService) { + this.authorsService = authorsService; + } + + @GetMapping("streamDownload") + public void streamDownload(HttpServletResponse response) + throws IOException { + long start = System.currentTimeMillis(); + authorsService.streamDownload(response); + log.info("streamDownload耗时:{}",(System.currentTimeMillis()-start)); + } + + @GetMapping("traditionDownload") + public void traditionDownload(HttpServletResponse response) + throws IOException { + long start = System.currentTimeMillis(); + authorsService.traditionDownload(response); + log.info("traditionDownload耗时:{}",(System.currentTimeMillis()-start)); + } + + + +} diff --git a/exportAndStreambijiao/src/main/java/com/woniu/service/Authors.java b/exportAndStreambijiao/src/main/java/com/woniu/service/Authors.java new file mode 100644 index 0000000..8357212 --- /dev/null +++ b/exportAndStreambijiao/src/main/java/com/woniu/service/Authors.java @@ -0,0 +1,71 @@ +package com.woniu.service; + +import java.sql.Date; + +public class Authors { + private String firstName; + + private String lastName; + + private String email; + + private Date birthdate; + + private Date added; + + private Integer id; + + public Integer getId() { + return id; + } + + public void setId(Integer id) { + this.id = id; + } + + public String getFirstName() { + return firstName; + } + + public void setFirstName(String firstName) { + this.firstName = firstName == null ? null : firstName.trim(); + } + + public String getLastName() { + return lastName; + } + + public void setLastName(String lastName) { + this.lastName = lastName == null ? null : lastName.trim(); + } + + public String getEmail() { + return email; + } + + public void setEmail(String email) { + this.email = email == null ? null : email.trim(); + } + + public Date getBirthdate() { + return birthdate; + } + + public void setBirthdate(Date birthdate) { + this.birthdate = birthdate; + } + + public Date getAdded() { + return added; + } + + public void setAdded(Date added) { + this.added = added; + } + + @Override + public String toString() { + return this.id + "," + this.firstName + "," + this.lastName + "," + this.email + "," + this.birthdate + "," + this.added; + } + +} diff --git a/exportAndStreambijiao/src/main/java/com/woniu/service/AuthorsMapper.java b/exportAndStreambijiao/src/main/java/com/woniu/service/AuthorsMapper.java new file mode 100644 index 0000000..3b07aa6 --- /dev/null +++ b/exportAndStreambijiao/src/main/java/com/woniu/service/AuthorsMapper.java @@ -0,0 +1,19 @@ +package com.woniu.service; + +import org.apache.ibatis.annotations.Mapper; +import org.apache.ibatis.annotations.Param; + +import java.util.List; + +/** + * @className: AuthorsMapper + * @author: Kevin + * @date: 2023/3/19 + **/ +@Mapper +public interface AuthorsMapper { + List selectByExample(); + + List streamByExample(); //以stream形式从mysql获取数据 + +} diff --git a/exportAndStreambijiao/src/main/java/com/woniu/service/AuthorsService.java b/exportAndStreambijiao/src/main/java/com/woniu/service/AuthorsService.java new file mode 100644 index 0000000..5f04e8c --- /dev/null +++ b/exportAndStreambijiao/src/main/java/com/woniu/service/AuthorsService.java @@ -0,0 +1,52 @@ +package com.woniu.service; + +import lombok.extern.slf4j.Slf4j; +import org.mybatis.spring.SqlSessionTemplate; +import org.springframework.stereotype.Service; + +import javax.servlet.http.HttpServletResponse; +import java.io.IOException; +import java.util.List; + +@Service +@Slf4j +public class AuthorsService { + private final SqlSessionTemplate sqlSessionTemplate; + private final AuthorsMapper authorsMapper; + + public AuthorsService(SqlSessionTemplate sqlSessionTemplate, AuthorsMapper authorsMapper) { + this.sqlSessionTemplate = sqlSessionTemplate; + this.authorsMapper = authorsMapper; + } + + /** + * stream读数据写文件方式 + * + * @param httpServletResponse + * @throws IOException + */ + public void streamDownload(HttpServletResponse httpServletResponse) + throws IOException { + + CustomResultHandler customResultHandler = new CustomResultHandler(new DownloadProcessor(httpServletResponse)); + sqlSessionTemplate.select( + "com.woniu.service.AuthorsMapper.streamByExample", customResultHandler); + httpServletResponse.getWriter().flush(); + httpServletResponse.getWriter().close(); + } + + /** + * 传统下载方式 + * + * @param httpServletResponse + * @throws IOException + */ + public void traditionDownload(HttpServletResponse httpServletResponse) + throws IOException { + List authors = authorsMapper.selectByExample(); + DownloadProcessor downloadProcessor = new DownloadProcessor(httpServletResponse); + authors.forEach(downloadProcessor::processData); + httpServletResponse.getWriter().flush(); + httpServletResponse.getWriter().close(); + } +} diff --git a/exportAndStreambijiao/src/main/java/com/woniu/service/CustomResultHandler.java b/exportAndStreambijiao/src/main/java/com/woniu/service/CustomResultHandler.java new file mode 100644 index 0000000..2c7d5a3 --- /dev/null +++ b/exportAndStreambijiao/src/main/java/com/woniu/service/CustomResultHandler.java @@ -0,0 +1,21 @@ +package com.woniu.service; + +import org.apache.ibatis.session.ResultContext; +import org.apache.ibatis.session.ResultHandler; + +public class CustomResultHandler implements ResultHandler { + private final DownloadProcessor downloadProcessor; + + public CustomResultHandler( + DownloadProcessor downloadProcessor) { + super(); + this.downloadProcessor = downloadProcessor; + } + + @Override + public void handleResult(ResultContext resultContext) { + Authors authors = (Authors) resultContext.getResultObject(); + downloadProcessor.processData(authors); + } + +} diff --git a/exportAndStreambijiao/src/main/java/com/woniu/service/DownloadProcessor.java b/exportAndStreambijiao/src/main/java/com/woniu/service/DownloadProcessor.java new file mode 100644 index 0000000..ee3aa60 --- /dev/null +++ b/exportAndStreambijiao/src/main/java/com/woniu/service/DownloadProcessor.java @@ -0,0 +1,27 @@ +package com.woniu.service; + + +import javax.servlet.http.HttpServletResponse; +import java.io.IOException; + +public class DownloadProcessor { + private final HttpServletResponse response; + + public DownloadProcessor(HttpServletResponse response) { + this.response = response; + String fileName = System.currentTimeMillis() + ".csv"; + this.response.addHeader("Content-Type", "application/csv"); + this.response.addHeader("Content-Disposition", "attachment; filename="+fileName); + this.response.setCharacterEncoding("UTF-8"); + } + + public void processData(E record) { + try { + response.getWriter().write(record.toString()); //如果是要写入csv,需要重写toString,属性通过","分割 + response.getWriter().write("\n"); + }catch (IOException e){ + e.printStackTrace(); + } + } + +} diff --git a/exportAndStreambijiao/src/main/resources/application.properties b/exportAndStreambijiao/src/main/resources/application.properties new file mode 100644 index 0000000..50fb150 --- /dev/null +++ b/exportAndStreambijiao/src/main/resources/application.properties @@ -0,0 +1,14 @@ + +#mybatis.config-location=classpath:/mybatis-config.xml + +#context.initializer.classes +#context.initializer.classes=com.fanfu.config.MyApplicationContextInitializer + +server.port=8088 +mybatis-plus.mapper-locations=classpath*:/mapper/*.xml + + +spring.datasource.url=jdbc:mysql://127.0.0.1:3306/test?useUnicode=true&characterEncoding=UTF-8&useSSL=false&serverTimezone=Asia/Shanghai +spring.datasource.username=root +spring.datasource.password=123456 +spring.datasource.driver-class-name=com.mysql.cj.jdbc.Driver \ No newline at end of file diff --git a/exportAndStreambijiao/src/main/resources/mapper/AuthorsMapper.xml b/exportAndStreambijiao/src/main/resources/mapper/AuthorsMapper.xml new file mode 100644 index 0000000..3b4c2fb --- /dev/null +++ b/exportAndStreambijiao/src/main/resources/mapper/AuthorsMapper.xml @@ -0,0 +1,21 @@ + + + + + + + + + + + + \ No newline at end of file diff --git a/exportAndStreambijiao/src/main/resources/sql.sql b/exportAndStreambijiao/src/main/resources/sql.sql new file mode 100644 index 0000000..4a98c46 --- /dev/null +++ b/exportAndStreambijiao/src/main/resources/sql.sql @@ -0,0 +1,9 @@ +CREATE TABLE `authors` ( + `id` int NOT NULL AUTO_INCREMENT, + `first_name` varchar(50) CHARACTER SET utf8mb3 COLLATE utf8mb3_unicode_ci NOT NULL, + `last_name` varchar(50) CHARACTER SET utf8mb3 COLLATE utf8mb3_unicode_ci NOT NULL, + `email` varchar(100) CHARACTER SET utf8mb3 COLLATE utf8mb3_unicode_ci NOT NULL, + `birthdate` date NOT NULL, + `added` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (`id`) +) ENGINE=InnoDB AUTO_INCREMENT=10095 DEFAULT CHARSET=utf8mb3 COLLATE=utf8mb3_unicode_ci \ No newline at end of file diff --git a/exportAndStreambijiao/src/test/java/com/fanfu/WoNiuApplicationTests.java b/exportAndStreambijiao/src/test/java/com/fanfu/WoNiuApplicationTests.java new file mode 100644 index 0000000..016b142 --- /dev/null +++ b/exportAndStreambijiao/src/test/java/com/fanfu/WoNiuApplicationTests.java @@ -0,0 +1,16 @@ +package com.fanfu; + + +import org.junit.Test; +import org.springframework.boot.test.context.SpringBootTest; + +@SpringBootTest +public class WoNiuApplicationTests { + + + + @Test + public void test(){ + + } +} diff --git a/exportAndStreambijiao/woniu-web.iml b/exportAndStreambijiao/woniu-web.iml new file mode 100644 index 0000000..431c31e --- /dev/null +++ b/exportAndStreambijiao/woniu-web.iml @@ -0,0 +1,110 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file