NM-gaoshuguang
/
BatchInsert
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

123

Browse Source
main
gaoshuguang 1 year ago
parent e054334e04
commit 1a410ad881
23 changed files with 450 additions and 2 deletions
Show Stats Download Patch File Download Diff File

6
.idea/vcs.xml
Unescape Escape View File

@ -0,0 +1,6 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="VcsDirectoryMappings">
<mapping directory="" vcs="Git" />
</component>
</project>

9
pom.xml
Unescape Escape View File

@ -109,7 +109,8 @@
<dependency>
<groupId>com.alibaba</groupId>
<artifactId>fastjson</artifactId>
<version>2.0.4</version>
<!-- <version>2.0.4</version>-->
<version>2.0.23</version>
</dependency>
<dependency>
<groupId>commons-io</groupId>
@ -226,7 +227,11 @@
<artifactId>mysql-connector-java</artifactId>
<version>5.1.46</version>
</dependency>
<dependency>
<groupId>cn.hutool</groupId>
<artifactId>hutool-all</artifactId>
<version>5.7.16</version>
</dependency>
</dependencies>
<dependencyManagement>

27
src/main/java/com/nmgs/entity/Detail.java
Unescape Escape View File

@ -0,0 +1,27 @@
package com.nmgs.entity;
import com.fasterxml.jackson.annotation.JsonProperty;
import lombok.Data;
/**
* @author: shuguang
* @date: 2024年09月02日 15:38
* @description:
*/
@Data
public class Detail {
@JsonProperty("roadno")
private int roadno;
@JsonProperty("feeunitid")
private String feeunitid;
@JsonProperty("fee")
private Long fee;
@JsonProperty("stano")
private int stano;
@JsonProperty("provinceId")
private String provinceId;
}

19
src/main/java/com/nmgs/entity/DivdataTestSta.java
Unescape Escape View File

@ -0,0 +1,19 @@
package com.nmgs.entity;
import lombok.Data;
import java.sql.Date;
/**
* @author: shuguang
* @date: 2024年09月02日 16:23
* @description:
*/
@Data
public class DivdataTestSta {
private Date CLASSDATE;
private Integer VEHICLETYPE;
private Long FEE;
private Long CARCOUNT;
private Long LENGTH;
}

24
src/main/java/com/nmgs/entity/RootObject.java
Unescape Escape View File

@ -0,0 +1,24 @@
package com.nmgs.entity;
import com.fasterxml.jackson.annotation.JsonProperty;
import lombok.Data;
import java.util.List;
/**
* @author: shuguang
* @date: 2024年09月02日 15:39
* @description:
*/
@Data
public class RootObject {
@JsonProperty("allid")
private String allid;
@JsonProperty("divType")
private int divType;
@JsonProperty("allfee")
private String allfee;
@JsonProperty("detail")
private List<Detail> details;
}

112
src/main/java/com/nmgs/test/Test0902.java
Unescape Escape View File

@ -0,0 +1,112 @@
package com.nmgs.test;
import cn.hutool.json.JSONArray;
import com.alibaba.fastjson2.JSON;
import com.alibaba.fastjson2.JSONObject;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.nmgs.entity.Detail;
import com.nmgs.entity.DivdataTestSta;
import com.nmgs.entity.RootObject;
import com.nmgs.util.DBUtil;
import com.nmgs.util.DateTimeUtil;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.jdbc.core.RowMapper;
import java.sql.Date;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Timestamp;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
/**
* @author: shuguang
* @date: 2024年09月02日 14:41
* @description:
*/
public class Test0902 {
public static void main(String[] args) {
final List<Integer> RoadNoList = Arrays.asList(10,15,11,20,24,25,26,27,28,37,38,39,42,43,46,48,49,50,53,54,60,61,62,63,64,65,66,67,81,84,85,86,87,88,89,90);
try {
for (int i = 0; i < 760; i++) {
//CLASSDATE,FEEUNITID,VEHICLETYPE,FEE,CARCOUNT LENGTH DIVDATA_TEST_STA
//String sql = "select * from divdata_test where rownum=1";
String sql = "select ID,CLASSDATE,DIVDETAIL,VEHICLETYPE,STATUS from divdata_test where rownum<=10000 and status=0";
JdbcTemplate jdbcTemplate = new JdbcTemplate(new DBUtil().Hikar_DataSource_GT());
List<Map<String, Object>> list = jdbcTemplate.queryForList(sql);
System.out.println("第"+(i+1)+"次处理，查询的集合条数" + list.size());
int count = 0;
if (list.size() > 0) {
for (Map<String, Object> map : list) {
String id = map.get("ID").toString();
String divdetail = map.get("DIVDETAIL").toString();
Timestamp classdate = (Timestamp) map.get("CLASSDATE");
Date date = new Date(classdate.getTime());
String formateString = DateTimeUtil.getFormateString(date, "yyyy-MM-dd");
int vehicletype = Integer.parseInt(map.get("VEHICLETYPE").toString());
String feeunitid = "";
long fee = 0;
long length = 0;
ObjectMapper objectMapper = new ObjectMapper();
RootObject rootObject = objectMapper.readValue(divdetail, RootObject.class);
List<Detail> details = rootObject.getDetails();
for (Detail detail : details) {
if (RoadNoList.contains(detail.getRoadno())) {
feeunitid = detail.getFeeunitid();
if("0".equals(feeunitid) ){
length=0;
fee=0;
}else {
length = jdbcTemplate.queryForObject("select length from nmgmpayadmin.FEEUNIT_TABLE where ID= ?", new Object[]{feeunitid}, Long.class);
fee = detail.getFee();
}
// 定义查询语句
String querySql = "SELECT FEE, LENGTH FROM DIVDATA_TEST_STA WHERE CLASSDATE = TO_DATE(?, 'yyyy-MM-dd') AND VEHICLETYPE = ?";
List<DivdataTestSta> results = jdbcTemplate.query(
querySql,
new Object[]{formateString, vehicletype},
(rs, rowNum) -> {
DivdataTestSta divdataTestSta = new DivdataTestSta();
divdataTestSta.setFEE(rs.getLong("FEE"));
divdataTestSta.setLENGTH(rs.getLong("LENGTH"));
return divdataTestSta;
}
);
if (!results.isEmpty()) {
// 更新记录
DivdataTestSta divdataTestSta = results.get(0);
long updatedFee = divdataTestSta.getFEE() + fee;
long updatedLength = divdataTestSta.getLENGTH() + length;
String updateSql = "UPDATE DIVDATA_TEST_STA SET FEE = ?, LENGTH = ?, CARCOUNT = CARCOUNT + 1 WHERE VEHICLETYPE = ? AND CLASSDATE = TO_DATE(?, 'yyyy-MM-dd')";
jdbcTemplate.update(updateSql, new Object[]{updatedFee, updatedLength, vehicletype, formateString});
} else {
// 插入新记录
String insertSql = "INSERT INTO DIVDATA_TEST_STA (CLASSDATE, VEHICLETYPE, FEE, LENGTH, CARCOUNT) VALUES (TO_DATE(?, 'yyyy-MM-dd'), ?, ?, ?, 1)";
jdbcTemplate.update(insertSql, new Object[]{formateString, vehicletype, fee, length});
}
}
}
count++;
String updateSql = "UPDATE DIVDATA_TEST SET STATUS = 1 WHERE ID = ? ";
jdbcTemplate.update(updateSql, new Object[]{id});
System.out.println("第"+(i+1)+"次处理，已处理条数" + count);
}
}
}
} catch (Exception e) {
e.printStackTrace();
}
}
}

66
src/main/java/com/nmgs/test/Test240902.java
Unescape Escape View File

@ -0,0 +1,66 @@
package com.nmgs.test;
import com.nmgs.util.BatchUtil;
import com.nmgs.util.BatchUtil2;
import com.nmgs.util.DBUtil;
import org.springframework.jdbc.core.JdbcTemplate;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
/**
* @author: shuguang
* @date: 2024年09月02日 17:18
* @description:
*/
public class Test240902 {
public static void main(String[] args) throws Exception{
JdbcTemplate jdbcTemplate = new JdbcTemplate(new DBUtil().Hikar_DataSource_GT());
long startAll = System.currentTimeMillis();
for (int j = 0; j < 710; j++) {
try {
String sql = "select ID,CLASSDATE,DIVDETAIL,VEHICLETYPE,STATUS from divdata_test where rownum<=10000 and status=0";
long start = System.currentTimeMillis();
List<Map<String, Object>> list = jdbcTemplate.queryForList(sql);
if (list.size() > 0) {
//int poolSize = Runtime.getRuntime().availableProcessors();
ExecutorService executorService = Executors.newFixedThreadPool(40);
// 将数据分成20份
int batchSize = list.size() / 40;
for (int i = 0; i < 40; i++) {
int startIndex = i * batchSize;
int endIndex = (i + 1) * batchSize;
if (i == 39) {
endIndex = list.size(); // 最后一份可能包含更多的数据
}
List<Map<String, Object>> subList = list.subList(startIndex, endIndex);
BatchUtil2 task = new BatchUtil2(jdbcTemplate, subList);
executorService.submit(task);
}
// 关闭线程池，尝试停止所有正在执行的任务
executorService.shutdown();
// 等待所有任务完成，设置合理的超时时间
executorService.awaitTermination(Long.MAX_VALUE, TimeUnit.NANOSECONDS);
}
long end = System.currentTimeMillis();
System.out.println("单次多线程处理耗时："+(end-start)+"毫秒");
} catch (Exception e) {
System.out.println("程序错误：" + e.getMessage());
}
}
long endAll = System.currentTimeMillis();
System.out.println("多线程处理总耗时："+(endAll-startAll)+"毫秒");
}
}

86
src/main/java/com/nmgs/test/Test241.java
Unescape Escape View File

@ -0,0 +1,86 @@
package com.nmgs.test;
import com.nmgs.util.DBUtil;
import org.springframework.jdbc.core.JdbcTemplate;
import java.util.List;
import java.util.Map;
/**
* @author: shuguang
* @date: 2024年07月12日 18:16
* @description:
*/
public class Test241 {
static {
ch.qos.logback.classic.LoggerContext loggerContext = (ch.qos.logback.classic.LoggerContext) org.slf4j.LoggerFactory.getILoggerFactory();
List<ch.qos.logback.classic.Logger> loggerList = loggerContext.getLoggerList();
loggerList.forEach(logger -> logger.setLevel(ch.qos.logback.classic.Level.INFO));
}
public static void main(String[] args) throws Exception {
String sql = "select ID from CT_GANTRYPASSID_JAVA WHERE TOLLPROVINCEID=150201" ;
JdbcTemplate jdbcTemplate = new JdbcTemplate(new DBUtil().Hikar_DataSource_MPAY());
List<Map<String, Object>> list = jdbcTemplate.queryForList(sql);
System.out.println("查询的集合条数"+list.size());
int count = 0;
if (list.size() > 0) {
for (Map<String, Object> map : list) {
String id = map.get("ID").toString();
String sql_data = "\n" +
"SELECT\n" +
"\t*\n" +
"FROM\n" +
"\t(select * from CT_GANTRYPASSID where ID = '" +
id +
"' and TOLLPROVINCEID = 150201 ) t1\n" +
"\tLEFT JOIN \n" +
"\t(select * from CT_GANTRYPASSID_JAVA where ID = '" +
id +
"' and TOLLPROVINCEID = 150201 )\n" +
"\t t2 ON t1.id = t2.id AND t1.TOLLPROVINCEID = t2.TOLLPROVINCEID \n" +
"\t \n" +
"\twhere 1=1 and (t1.EXITTRANSTYPE <> t2.EXITTRANSTYPE \n" +
"\tOR t1.SPLITTIME <> t2.SPLITTIME \n" +
"\tOR t1.PASSID <> t2.PASSID \n" +
"\tOR t1.EXTRANSACTIONID <> t2.EXTRANSACTIONID \n" +
"\tOR t1.EXTIME <> t2.EXTIME \n" +
"\tOR t1.EXTOLLSTATIONNAME <> t2.EXTOLLSTATIONNAME \n" +
"\tOR t1.EXTOLLLANEID <> t2.EXTOLLLANEID \n" +
"\tOR t1.EXFEE <> t2.EXFEE \n" +
"\tOR t1.FEE <> t2.FEE \n" +
"\tOR t1.SPLITFEE <> t2.SPLITFEE\n" +
"\tOR t1.SPLITRULE <> t2.SPLITRULE \n" +
"\tOR t1.EXITFEETYPE <> t2.EXITFEETYPE \n" +
"\tOR t1.VEHICLESIGN <> t2.VEHICLESIGN \n" +
"\tOR t1.TOLLINTERVALID <> t2.TOLLINTERVALID \n" +
"\tOR t1.TRANSNUM <> t2.TRANSNUM \n" +
"\tOR t1.SERPROVINCEID <> t2.SERPROVINCEID \n" +
"\tOR t1.PROCESSTIME <> t2.PROCESSTIME \n" +
"\tOR t1.VEHICLEID <> t2.VEHICLEID \n" +
"\tOR t1.CARDID <> t2.CARDID \n" +
"\tOR t1.MEDIATYPE <> t2.MEDIATYPE \n" +
"\tOR t1.MEDIANO <> t2.MEDIANO \n" +
"\tOR t1.ENTOLLLANEID <> t2.ENTOLLLANEID \n" +
"\tOR t1.VEHICLETYPE <> t2.VEHICLETYPE \n" +
"\tOR t1.BASICFILENAME <> t2.BASICFILENAME \n" +
"\tOR t1.APPOINTID <> t2.APPOINTID)";
List<Map<String, Object>> list_data = jdbcTemplate.queryForList(sql_data);
count++;
if (list_data.size() > 0) {
System.out.println("数据不一致ID：" + id);
}
System.out.println("已处理条数：" + count);
}
}
System.out.println("查询完成");
}
}

102
src/main/java/com/nmgs/util/BatchUtil2.java
Unescape Escape View File

@ -0,0 +1,102 @@
package com.nmgs.util;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.nmgs.entity.Detail;
import com.nmgs.entity.DivdataTestSta;
import com.nmgs.entity.RootObject;
import org.springframework.jdbc.core.JdbcTemplate;
import java.sql.Date;
import java.sql.Timestamp;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
/**
* @author: shuguang
* @date: 2024年09月02日 17:21
* @description:
*/
public class BatchUtil2 implements Runnable{
final List<Integer> RoadNoList = Arrays.asList(10,15,11,20,24,25,26,27,28,37,38,39,42,43,46,48,49,50,53,54,60,61,62,63,64,65,66,67,81,84,85,86,87,88,89,90);
private JdbcTemplate jdbcTemplate;
private List<Map<String, Object>> list;
public BatchUtil2(JdbcTemplate jdbcTemplate, List<Map<String, Object>> list) {
this.jdbcTemplate = jdbcTemplate;
this.list = list;
}
@Override
public void run() {
try {
if (list.size() > 0) {
for (Map<String, Object> map : list) {
String id = map.get("ID").toString();
String divdetail = map.get("DIVDETAIL").toString();
Timestamp classdate = (Timestamp) map.get("CLASSDATE");
Date date = new Date(classdate.getTime());
String formateString = DateTimeUtil.getFormateString(date, "yyyy-MM-dd");
int vehicletype = Integer.parseInt(map.get("VEHICLETYPE").toString());
String feeunitid = "";
long fee = 0;
long length = 0;
ObjectMapper objectMapper = new ObjectMapper();
RootObject rootObject = objectMapper.readValue(divdetail, RootObject.class);
List<Detail> details = rootObject.getDetails();
for (Detail detail : details) {
if (RoadNoList.contains(detail.getRoadno())) {
feeunitid = detail.getFeeunitid();
if("0".equals(feeunitid) ){
length=0;
fee=0;
}else {
length = jdbcTemplate.queryForObject("select length from nmgmpayadmin.FEEUNIT_TABLE where ID= ?", new Object[]{feeunitid}, Long.class);
fee = detail.getFee();
}
// 定义查询语句
String querySql = "SELECT FEE, LENGTH FROM DIVDATA_TEST_STA WHERE CLASSDATE = TO_DATE(?, 'yyyy-MM-dd') AND VEHICLETYPE = ?";
List<DivdataTestSta> results = jdbcTemplate.query(
querySql,
new Object[]{formateString, vehicletype},
(rs, rowNum) -> {
DivdataTestSta divdataTestSta = new DivdataTestSta();
divdataTestSta.setFEE(rs.getLong("FEE"));
divdataTestSta.setLENGTH(rs.getLong("LENGTH"));
return divdataTestSta;
}
);
if (!results.isEmpty()) {
// 更新记录
DivdataTestSta divdataTestSta = results.get(0);
long updatedFee = divdataTestSta.getFEE() + fee;
long updatedLength = divdataTestSta.getLENGTH() + length;
String updateSql = "UPDATE DIVDATA_TEST_STA SET FEE = ?, LENGTH = ?, CARCOUNT = CARCOUNT + 1 WHERE VEHICLETYPE = ? AND CLASSDATE = TO_DATE(?, 'yyyy-MM-dd')";
jdbcTemplate.update(updateSql, new Object[]{updatedFee, updatedLength, vehicletype, formateString});
} else {
// 插入新记录
String insertSql = "INSERT INTO DIVDATA_TEST_STA (CLASSDATE, VEHICLETYPE, FEE, LENGTH, CARCOUNT) VALUES (TO_DATE(?, 'yyyy-MM-dd'), ?, ?, ?, 1)";
jdbcTemplate.update(insertSql, new Object[]{formateString, vehicletype, fee, length});
}
}
}
String updateSql = "UPDATE DIVDATA_TEST SET STATUS = 1 WHERE ID = ? ";
jdbcTemplate.update(updateSql, new Object[]{id});
}
}
} catch (JsonProcessingException e) {
e.printStackTrace();
}
}
}

1
src/main/java/com/nmgs/util/PlateRegexutil.java
Unescape Escape View File

@ -27,6 +27,7 @@ public class PlateRegexutil {
for (int j = 0; j < regex.length; j++) {
if (plateid.matches(regex[j])) {
relseltflag= true;
break;
}
}
return relseltflag;

BIN
target/classes/com/nmgs/entity/Detail.class
View File

Binary file not shown.

BIN
target/classes/com/nmgs/entity/DivdataTestSta.class
View File

Binary file not shown.

BIN
target/classes/com/nmgs/entity/RootObject.class
View File

Binary file not shown.

BIN
target/classes/com/nmgs/test/DataTableComparisonTest.class
View File

Binary file not shown.

BIN
target/classes/com/nmgs/test/Test0902.class
View File

Binary file not shown.

BIN
target/classes/com/nmgs/test/Test240902.class
View File

Binary file not shown.

BIN
target/classes/com/nmgs/test/Test241.class
View File

Binary file not shown.

BIN
target/classes/com/nmgs/test/sta_test.class
View File

Binary file not shown.

BIN
target/classes/com/nmgs/test/sta_test1.class
View File

Binary file not shown.

BIN
target/classes/com/nmgs/util/BatchUtil.class
View File

Binary file not shown.

BIN
target/classes/com/nmgs/util/BatchUtil2.class
View File

Binary file not shown.

BIN
target/classes/com/nmgs/util/DBUtil.class
View File

Binary file not shown.

BIN
target/classes/com/nmgs/util/PlateRegexutil.class
View File

Binary file not shown.
Write Preview
Loading…
Cancel
Save