请帮我实现使用多线程进行数据库插入大量数据的java代码
时间: 2024-05-10 15:18:37 浏览: 286
一个多线程访问数据库的代码
4星 · 用户满意度95%
好的,以下是使用多线程进行数据库插入大量数据的Java代码示例:
```java
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
public class InsertDataMultiThread {
private static final String DB_DRIVER = "com.mysql.jdbc.Driver";
private static final String DB_CONNECTION = "jdbc:mysql://localhost:3306/mydatabase";
private static final String DB_USER = "root";
private static final String DB_PASSWORD = "password";
private static final int THREAD_POOL_SIZE = 10;
private static final int BATCH_SIZE = 1000;
public static void main(String[] args) {
Connection dbConnection = null;
try {
// 获取数据库连接
dbConnection = getDBConnection();
// 设置自动提交为 false
dbConnection.setAutoCommit(false);
// 创建一个线程池
ExecutorService executor = Executors.newFixedThreadPool(THREAD_POOL_SIZE);
// 创建一个 PreparedStatement 对象
PreparedStatement preparedStatement = dbConnection.prepareStatement("INSERT INTO mytable (col1, col2, col3) VALUES (?, ?, ?)");
// 构造数据列表
List<Data> dataList = new ArrayList<>();
for (int i = 0; i < 1000000; i++) {
Data data = new Data("value1", "value2", "value3");
dataList.add(data);
}
// 将数据列表分割成多个批次
List<List<Data>> batches = splitIntoBatches(dataList, BATCH_SIZE);
// 依次将每个批次提交到线程池中执行
for (List<Data> batch : batches) {
Runnable task = new InsertDataTask(preparedStatement, batch);
executor.execute(task);
}
// 关闭线程池
executor.shutdown();
// 等待所有任务执行完毕
while (!executor.isTerminated()) {}
// 提交事务
dbConnection.commit();
System.out.println("Data inserted successfully.");
} catch (SQLException e) {
System.err.println(e.getMessage());
} finally {
try {
if (dbConnection != null) {
// 恢复自动提交
dbConnection.setAutoCommit(true);
// 关闭数据库连接
dbConnection.close();
}
} catch (SQLException e) {
System.err.println(e.getMessage());
}
}
}
private static Connection getDBConnection() throws SQLException {
Connection dbConnection = null;
try {
Class.forName(DB_DRIVER);
} catch (ClassNotFoundException e) {
System.err.println(e.getMessage());
}
dbConnection = DriverManager.getConnection(DB_CONNECTION, DB_USER, DB_PASSWORD);
return dbConnection;
}
private static <T> List<List<T>> splitIntoBatches(List<T> list, int batchSize) {
List<List<T>> batches = new ArrayList<>();
int size = list.size();
for (int i = 0; i < size; i += batchSize) {
batches.add(list.subList(i, Math.min(size, i + batchSize)));
}
return batches;
}
private static class InsertDataTask implements Runnable {
private PreparedStatement preparedStatement;
private List<Data> dataList;
public InsertDataTask(PreparedStatement preparedStatement, List<Data> dataList) {
this.preparedStatement = preparedStatement;
this.dataList = dataList;
}
@Override
public void run() {
try {
for (Data data : dataList) {
preparedStatement.setString(1, data.getCol1());
preparedStatement.setString(2, data.getCol2());
preparedStatement.setString(3, data.getCol3());
preparedStatement.addBatch();
}
preparedStatement.executeBatch();
} catch (SQLException e) {
System.err.println(e.getMessage());
}
}
}
private static class Data {
private String col1;
private String col2;
private String col3;
public Data(String col1, String col2, String col3) {
this.col1 = col1;
this.col2 = col2;
this.col3 = col3;
}
public String getCol1() {
return col1;
}
public void setCol1(String col1) {
this.col1 = col1;
}
public String getCol2() {
return col2;
}
public void setCol2(String col2) {
this.col2 = col2;
}
public String getCol3() {
return col3;
}
public void setCol3(String col3) {
this.col3 = col3;
}
}
}
```
在这个示例代码中,我们使用了线程池来并发执行大量的数据库插入操作。每个线程负责将一个批次的数据插入到数据库中。我们将数据列表分割成多个批次,每个批次的大小为 BATCH_SIZE,然后依次将每个批次提交到线程池中执行。在每个线程中,我们使用 PreparedStatement 对象来执行批量插入操作,可以有效地减少数据库连接和交互的次数。当所有任务执行完毕后,我们提交事务并关闭数据库连接。
如果你想要使用这个示例代码,需要将 DB_CONNECTION、DB_USER 和 DB_PASSWORD 替换成你自己的数据库连接信息,同时需要创建一个名为 mytable 的表,并且包含三个文本列 col1、col2 和 col3。
阅读全文