一:介绍
1 | 为了防止日志泄露造成的敏感信息外露出去,比如:姓名,手机号,身份证号, |
二:效果
脱敏前
1
2
3
4
52024-08-22 12:18:14.121 [http-nio-10047-exec-2] INFO c.vji.demo2.controller.ApiController [26] - data:{"msg":"success","code":1,"idCard":"41140219930989876X","name":"李四","mobile":"17898981762"}
2024-08-22 12:18:14.121 [http-nio-10047-exec-2] INFO c.vji.demo2.controller.ApiController [27] - customerName:张三
2024-08-22 12:18:14.122 [http-nio-10047-exec-2] INFO c.vji.demo2.controller.ApiController [28] - mobile:13178787822
2024-08-22 12:18:14.122 [http-nio-10047-exec-2] INFO c.vji.demo2.controller.ApiController [29] - address:上海市黄浦区
2024-08-22 12:18:14.122 [http-nio-10047-exec-2] INFO c.vji.demo2.controller.ApiController [30] - idCard:41140219930989876X脱敏后
1
2
3
4
52024-08-22 12:18:36.372 [http-nio-10047-exec-1] INFO c.vji.demo2.controller.ApiController [26] - data:{"msg":"success","code":1,"idCard":"411402************","name":"李*","mobile":"178****1762"}
2024-08-22 12:18:36.375 [http-nio-10047-exec-1] INFO c.vji.demo2.controller.ApiController [27] - customerName:张*
2024-08-22 12:18:36.376 [http-nio-10047-exec-1] INFO c.vji.demo2.controller.ApiController [28] - mobile:131****7822
2024-08-22 12:18:36.376 [http-nio-10047-exec-1] INFO c.vji.demo2.controller.ApiController [29] - address:上海市***
2024-08-22 12:18:36.378 [http-nio-10047-exec-1] INFO c.vji.demo2.controller.ApiController [30] - idCard:411402************
三:springboot+log4j2实现日志脱敏
方案
1
自定义PatternLayout,重写序列化方法然后执行正则筛选然后脱敏处理。
MyPatternLayout.java
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205package com.vji.demo1.config;
import java.nio.charset.Charset;
import java.util.Arrays;
import java.util.HashMap;
import java.util.Map;
import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import org.apache.commons.collections4.MapUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.logging.log4j.core.Layout;
import org.apache.logging.log4j.core.LogEvent;
import org.apache.logging.log4j.core.config.Node;
import org.apache.logging.log4j.core.config.plugins.Plugin;
import org.apache.logging.log4j.core.config.plugins.PluginAttribute;
import org.apache.logging.log4j.core.config.plugins.PluginFactory;
import org.apache.logging.log4j.core.layout.AbstractStringLayout;
import org.apache.logging.log4j.core.layout.PatternLayout;
import com.alibaba.fastjson.JSON;
import lombok.extern.slf4j.Slf4j;
/**
* @author RenJie
*/
@Plugin(name = "MyPatternLayout", category = Node.CATEGORY, elementType = Layout.ELEMENT_TYPE, printObject = true)
@Slf4j
public class MyPatternLayout extends AbstractStringLayout {
private final PatternLayout patternLayout;
protected MyPatternLayout(Charset charset, String pattern) {
super(charset);
//这里是为了使用原本layout携带的格式
this.patternLayout = PatternLayout.newBuilder().withPattern(pattern).build();
//初始化脱敏规则
initRule();
}
@Override
public String toSerializable(LogEvent event) {
// 实现你的自定义逻辑来格式化LogEvent
// 例如:
// return "自定义格式: " + event.getMessage().getFormattedMessage() + "\n";
// return "自定义:" + patternLayout.toSerializable(event);
//脱敏处理
return hideMarkLog(patternLayout.toSerializable(event));
}
// 工厂方法,用于在配置文件中创建CustomLayout实例
@PluginFactory
public static MyPatternLayout createLayout(
@PluginAttribute(value = "charset", defaultString = "UTF-8") Charset charset,
@PluginAttribute(value = "pattern", defaultString = "[%d{yyyy-MM-dd HH:mm:ss}] %-5level %logger{36} - %msg%n") String pattern
// 可以添加其他配置属性
) {
// 这里可以根据需要添加对配置属性的处理
return new MyPatternLayout(charset, pattern);
}
/**
* 要匹配的正则表达式map
*/
private static final Map<String, Pattern> REG_PATTERN_MAP = new HashMap<>();
private static final Map<String, String> KEY_REG_MAP = new HashMap<>();
private void initRule() {
try {
if (MapUtils.isEmpty(Log4j2Rule.regularMap)) {
return;
}
Log4j2Rule.regularMap.forEach((a, b) -> {
if (StringUtils.isNotBlank(a)) {
Map<String, String> collect = Arrays.stream(a.split(","))
.collect(Collectors.toMap(c -> c, w -> b, (key1, key2) -> key1));
KEY_REG_MAP.putAll(collect);
}
Pattern compile = Pattern.compile(b);
REG_PATTERN_MAP.put(b, compile);
});
log.info("KEY_REG_MAP:{}", JSON.toJSONString(KEY_REG_MAP));
log.info("REG_PATTERN_MAP:{}", JSON.toJSONString(REG_PATTERN_MAP));
} catch (Exception e) {
log.info(">>>>>> 初始化日志脱敏规则失败 ERROR:{0}", e);
}
}
/**
* 处理日志信息,进行脱敏 1.判断配置文件中是否已经配置需要脱敏字段 2.判断内容是否有需要脱敏的敏感信息 2.1 没有需要脱敏信息直接返回 2.2 处理:
* 身份证 ,姓名,手机号,地址敏感信息
*/
public String hideMarkLog(String logStr) {
try {
// 1.判断配置文件中是否已经配置需要脱敏字段
if (StringUtils.isBlank(logStr) || MapUtils.isEmpty(KEY_REG_MAP) || MapUtils.isEmpty(REG_PATTERN_MAP)) {
return logStr;
}
// 2.判断内容是否有需要脱敏的敏感信息
Set<String> charKeys = KEY_REG_MAP.keySet();
for (String key : charKeys) {
if (logStr.contains(key)) {
String regExp = KEY_REG_MAP.get(key);
logStr = matchingAndEncrypt(logStr, regExp, key);
}
}
return logStr;
} catch (Exception e) {
log.info(">>>>>>>>> 脱敏处理异常 ERROR:{0}", e);
// 如果抛出异常为了不影响流程,直接返回原信息
return logStr;
}
}
/**
* 正则匹配对应的对象。
*
* @param msg 日志对象
* @param regExp 正则匹配
* @param key 字段
* @return 找到对应对象
*/
private static String matchingAndEncrypt(String msg, String regExp, String key) {
Pattern pattern = Pattern.compile(regExp);
Matcher matcher = pattern.matcher(msg);
int length = key.length() + 5;
boolean names = Log4j2Rule.USER_NAME_STR.contains(key);
boolean address = Log4j2Rule.USER_ADDRESS_STR.contains(key);
boolean phones = Log4j2Rule.USER_PHONE_STR.contains(key);
boolean idcard = Log4j2Rule.USER_IDCARD_STR.contains(key);
String hiddenStr = "";
StringBuffer result = new StringBuffer(msg);
int i = 0;
while (matcher.find()) {
String originStr = matcher.group();
// 计算关键词和需要脱敏词的距离小于5。
i = msg.indexOf(originStr, i);
if (i < 0) {
continue;
}
int span = i - length;
int startIndex = Math.max(span, 0);
String substring = msg.substring(startIndex, i);
if (StringUtils.isBlank(substring) || !substring.contains(key)) {
i += key.length();
continue;
}
if (names) {
hiddenStr = hideMarkStr(originStr, 1);
} else if (phones) {
hiddenStr = hideMarkStr(originStr, 2);
} else if (address) {
hiddenStr = hideMarkStr(originStr, 3);
} else if (idcard) {
hiddenStr = hideMarkStr(originStr, 4);
}
msg = result.replace(i, i + originStr.length(), hiddenStr).toString();
}
return msg;
}
/**
* 标记敏感文字规则
*
* @param needHideMark 日志对象
* @param type 脱敏规则类型 1为姓名脱敏规则,2为姓名脱敏规则手机号,3为地址脱敏规则,4为身份证脱敏规则
* @return 返回过敏字段
*/
private static String hideMarkStr(String needHideMark, int type) {
if (StringUtils.isBlank(needHideMark)) {
return "";
}
int startSize = 0, endSize = 0, mark = 0, length = needHideMark.length();
StringBuffer hideRegBuffer = new StringBuffer("(\\S{");
StringBuffer replaceSb = new StringBuffer("$1");
if (type == 1) {
startSize = 1;
} else if (type == 2) {
int i = length / 3;
startSize = i;
endSize = i + 1;
} else if (type == 3) {
startSize = 3;
} else if (type == 4) {
startSize = 6;
}
mark = length - startSize - endSize;
for (int i = 0; i < mark; i++) {
replaceSb.append("*");
}
hideRegBuffer.append(startSize).append("})\\S*(\\S{").append(endSize).append("})");
replaceSb.append("$2");
needHideMark = needHideMark.replaceAll(hideRegBuffer.toString(), replaceSb.toString());
return needHideMark;
}
}Log4j2Rule.java
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50package com.vji.demo1.config;
import java.util.HashMap;
import java.util.Map;
/**
* 拦截脱敏的日志:
* 1,身份证
* 2,姓名
* 3,身份证号 (目前暂时不脱敏)
* 4、地址
* 脱敏规则后续可以优化在配置文件中
*
**/
public class Log4j2Rule {
/**
* 正则匹配 关键词 类别
*/
public static Map<String, String> regularMap = new HashMap<>();
//用戶名字段名
public static final String USER_NAME_STR = "name,userName,petName,nickName,customerName";
//地址 字段名
public static final String USER_ADDRESS_STR = "receiveAddress,address,order_address";
//身份证 字段名
public static final String USER_IDCARD_STR = "idCard,cardNumber";
//手机号 字段名
public static final String USER_PHONE_STR = "mobile,phone,phoneNum,phoneNumber,cellPhone,agentMobile,telephone";
/**
* 正则匹配,根据业务要求自定义
*/
//身份证正则
private static final String IDCARD_REGEXP = "(\\d{17}[0-9Xx]|\\d{14}[0-9Xx])";
//用户名正则
private static final String USERNAME_REGEXP = "[\\u4e00-\\u9fa5_a-zA-Z0-9-]{2,50}";
//地址正则
private static final String ADDRESS_REGEXP = "[\\u4e00-\\u9fa5_a-zA-Z0-9-]{3,200}";
//手机号正则
private static final String PHONE_REGEXP = "(?<!\\d)(?:(?:1[3456789]\\d{9})|(?:861[356789]\\d{9}))(?!\\d)";
static {
regularMap.put(USER_NAME_STR, USERNAME_REGEXP);
regularMap.put(USER_ADDRESS_STR, ADDRESS_REGEXP);
regularMap.put(USER_IDCARD_STR, IDCARD_REGEXP);
regularMap.put(USER_PHONE_STR, PHONE_REGEXP);
}
}log4j2.xml
1
2
3
4
5
6
7
8
9
10
11
12
13
14<?xml version="1.0" encoding="UTF-8"?>
<Configuration status="WARN">
<Layout type="com.vji.demo1.config.MyPatternLayout" charset="UTF-8"/>
<Appenders>
<Console name="Console" target="SYSTEM_OUT">
<MyPatternLayout charset="UTF-8" pattern="%d{yyyy-MM-dd HH:mm:ss} [%t] %-5level %logger{36} - %msg%n"/>
</Console>
</Appenders>
<Loggers>
<Root level="info">
<AppenderRef ref="Console"/>
</Root>
</Loggers>
</Configuration>
1 | 主要调整如下两点修改,网上别的地方可能只说修改第二个框的内容, |
- log4j2.xml版本2
1
2
3
4
5
6
7
8
9
10
11
12
13<?xml version="1.0" encoding="UTF-8"?>
<Configuration status="WARN" packages="com.vji.demo1.config">
<Appenders>
<Console name="Console" target="SYSTEM_OUT">
<MyPatternLayout charset="UTF-8" pattern="%d{yyyy-MM-dd HH:mm:ss} [%t] %-5level %logger{36} - %msg%n"/>
</Console>
</Appenders>
<Loggers>
<Root level="info">
<AppenderRef ref="Console"/>
</Root>
</Loggers>
</Configuration>
1 | 在Configuration中把自己定义的插件包含进来,然后去掉Layout标签即可。 |
- 测试
1
2测试可以使用ApiController.java中的log方法,地址是:
http://127.0.0.1:10046/demo1/api/log
四:springboot+logback实现日志脱敏
方案
1
自定义PatternLayout,拦截日志输出内容然后执行正则筛选然后脱敏处理。
MyPatternLayout.java
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171package com.vji.demo2.config;
import java.util.Arrays;
import java.util.HashMap;
import java.util.Map;
import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import org.apache.commons.collections4.MapUtils;
import org.apache.commons.lang3.StringUtils;
import com.alibaba.fastjson.JSON;
import ch.qos.logback.classic.PatternLayout;
import ch.qos.logback.classic.spi.ILoggingEvent;
import lombok.extern.slf4j.Slf4j;
@Slf4j
public class MyPatternLayout extends PatternLayout {
/**
* 要匹配的正则表达式map
*/
private static final Map<String, Pattern> REG_PATTERN_MAP = new HashMap<>();
private static final Map<String, String> KEY_REG_MAP = new HashMap<>();
{
initRule();
}
@Override
public String doLayout(ILoggingEvent event) {
String message = super.doLayout(event);
//脱敏处理
return hideMarkLog(message);
}
private void initRule() {
try {
if (MapUtils.isEmpty(LogRule.regularMap)) {
return;
}
LogRule.regularMap.forEach((a, b) -> {
if (StringUtils.isNotBlank(a)) {
Map<String, String> collect = Arrays.stream(a.split(","))
.collect(Collectors.toMap(c -> c, w -> b, (key1, key2) -> key1));
KEY_REG_MAP.putAll(collect);
}
Pattern compile = Pattern.compile(b);
REG_PATTERN_MAP.put(b, compile);
});
log.info("KEY_REG_MAP:{}", JSON.toJSONString(KEY_REG_MAP));
log.info("REG_PATTERN_MAP:{}", JSON.toJSONString(REG_PATTERN_MAP));
} catch (Exception e) {
log.info(">>>>>> 初始化日志脱敏规则失败 ERROR:{0}", e);
}
}
/**
* 处理日志信息,进行脱敏 1.判断配置文件中是否已经配置需要脱敏字段 2.判断内容是否有需要脱敏的敏感信息 2.1 没有需要脱敏信息直接返回 2.2 处理:
* 身份证 ,姓名,手机号,地址敏感信息
*/
public String hideMarkLog(String logStr) {
try {
// 1.判断配置文件中是否已经配置需要脱敏字段
if (StringUtils.isBlank(logStr) || MapUtils.isEmpty(KEY_REG_MAP) || MapUtils.isEmpty(REG_PATTERN_MAP)) {
return logStr;
}
// 2.判断内容是否有需要脱敏的敏感信息
Set<String> charKeys = KEY_REG_MAP.keySet();
for (String key : charKeys) {
if (logStr.contains(key)) {
String regExp = KEY_REG_MAP.get(key);
logStr = matchingAndEncrypt(logStr, regExp, key);
}
}
return logStr;
} catch (Exception e) {
log.info(">>>>>>>>> 脱敏处理异常 ERROR:{0}", e);
// 如果抛出异常为了不影响流程,直接返回原信息
return logStr;
}
}
/**
* 正则匹配对应的对象。
*
* @param msg 日志对象
* @param regExp 正则匹配
* @param key 字段
* @return 找到对应对象
*/
private static String matchingAndEncrypt(String msg, String regExp, String key) {
Pattern pattern = Pattern.compile(regExp);
Matcher matcher = pattern.matcher(msg);
int length = key.length() + 5;
boolean names = LogRule.USER_NAME_STR.contains(key);
boolean address = LogRule.USER_ADDRESS_STR.contains(key);
boolean phones = LogRule.USER_PHONE_STR.contains(key);
boolean idcard = LogRule.USER_IDCARD_STR.contains(key);
String hiddenStr = "";
StringBuffer result = new StringBuffer(msg);
int i = 0;
while (matcher.find()) {
String originStr = matcher.group();
// 计算关键词和需要脱敏词的距离小于5。
i = msg.indexOf(originStr, i);
if (i < 0) {
continue;
}
int span = i - length;
int startIndex = Math.max(span, 0);
String substring = msg.substring(startIndex, i);
if (StringUtils.isBlank(substring) || !substring.contains(key)) {
i += key.length();
continue;
}
if (names) {
hiddenStr = hideMarkStr(originStr, 1);
} else if (phones) {
hiddenStr = hideMarkStr(originStr, 2);
} else if (address) {
hiddenStr = hideMarkStr(originStr, 3);
} else if (idcard) {
hiddenStr = hideMarkStr(originStr, 4);
}
msg = result.replace(i, i + originStr.length(), hiddenStr).toString();
}
return msg;
}
/**
* 标记敏感文字规则
*
* @param needHideMark 日志对象
* @param type 脱敏规则类型 1为姓名脱敏规则,2为姓名脱敏规则手机号,3为地址脱敏规则,4为身份证脱敏规则
* @return 返回过敏字段
*/
private static String hideMarkStr(String needHideMark, int type) {
if (StringUtils.isBlank(needHideMark)) {
return "";
}
int startSize = 0, endSize = 0, mark = 0, length = needHideMark.length();
StringBuffer hideRegBuffer = new StringBuffer("(\\S{");
StringBuffer replaceSb = new StringBuffer("$1");
if (type == 1) {
startSize = 1;
} else if (type == 2) {
int i = length / 3;
startSize = i;
endSize = i + 1;
} else if (type == 3) {
startSize = 3;
} else if (type == 4) {
startSize = 6;
}
mark = length - startSize - endSize;
for (int i = 0; i < mark; i++) {
replaceSb.append("*");
}
hideRegBuffer.append(startSize).append("})\\S*(\\S{").append(endSize).append("})");
replaceSb.append("$2");
needHideMark = needHideMark.replaceAll(hideRegBuffer.toString(), replaceSb.toString());
return needHideMark;
}
}logRule.java
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50package com.vji.demo2.config;
import java.util.HashMap;
import java.util.Map;
/**
* 拦截脱敏的日志:
* 1,身份证
* 2,姓名
* 3,身份证号 (目前暂时不脱敏)
* 4、地址
* 脱敏规则后续可以优化在配置文件中
*
**/
public class LogRule {
/**
* 正则匹配 关键词 类别
*/
public static Map<String, String> regularMap = new HashMap<>();
//用戶名字段名
public static final String USER_NAME_STR = "name,userName,petName,nickName,customerName";
//地址 字段名
public static final String USER_ADDRESS_STR = "receiveAddress,address,order_address";
//身份证 字段名
public static final String USER_IDCARD_STR = "idCard,cardNumber";
//手机号 字段名
public static final String USER_PHONE_STR = "mobile,phone,phoneNum,phoneNumber,cellPhone,agentMobile,telephone";
/**
* 正则匹配,根据业务要求自定义
*/
//身份证正则
private static final String IDCARD_REGEXP = "(\\d{17}[0-9Xx]|\\d{14}[0-9Xx])";
//用户名正则
private static final String USERNAME_REGEXP = "[\\u4e00-\\u9fa5_a-zA-Z0-9-]{2,50}";
//地址正则
private static final String ADDRESS_REGEXP = "[\\u4e00-\\u9fa5_a-zA-Z0-9-]{3,200}";
//手机号正则
private static final String PHONE_REGEXP = "(?<!\\d)(?:(?:1[3456789]\\d{9})|(?:861[356789]\\d{9}))(?!\\d)";
static {
regularMap.put(USER_NAME_STR, USERNAME_REGEXP);
regularMap.put(USER_ADDRESS_STR, ADDRESS_REGEXP);
regularMap.put(USER_IDCARD_STR, IDCARD_REGEXP);
regularMap.put(USER_PHONE_STR, PHONE_REGEXP);
}
}logback-spring.xml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29<?xml version="1.0" encoding="UTF-8"?>
<configuration debug="true">
<!-- 彩色日志 -->
<!-- 彩色日志依赖的渲染类 -->
<conversionRule conversionWord="clr"
converterClass="org.springframework.boot.logging.logback.ColorConverter" />
<conversionRule conversionWord="wex"
converterClass="org.springframework.boot.logging.logback.WhitespaceThrowableProxyConverter" />
<!-- 彩色日志格式 -->
<property name="CONSOLE_LOG_PATTERN"
value="%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{36} [%line] - %msg%n" />
<!--输出到控制台 -->
<appender name="CONSOLE"
class="ch.qos.logback.core.ConsoleAppender">
<encoder class="ch.qos.logback.core.encoder.LayoutWrappingEncoder">
<Layout class="com.vji.demo2.config.MyPatternLayout">
<pattern>${CONSOLE_LOG_PATTERN}</pattern>
</Layout>
<charset>UTF-8</charset>
</encoder>
</appender>
<root level="info">
<appender-ref ref="CONSOLE" />
</root>
</configuration>
1 | 修改两点,encoder加行class="ch.qos.logback.core.encoder.LayoutWrappingEncoder", |
- 测试
1
2测试可以使用ApiController.java中的log方法,地址是:
http://127.0.0.1:10047/demo2/api/log
五:总结
1 | 网上有很多例子,可能由于某些版本问题亲测下来不能使用,我这里使用的是 |
- 结果
*************感谢您的阅读*************