leftyer/SamWaf
1
0
Fork 0
mirror of https://gitee.com/samwaf/SamWaf.git synced 2025-12-06 06:58:54 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix:home counter

Browse Source 
#IBHQDZ
This commit is contained in:
samwaf
2025-01-20 10:48:05 +08:00
parent 13848134d1
commit ebcf3c16cc
7 changed files with 81 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
global/global.go
View File

@@ -66,6 +66,7 @@ var (
GWAF_RELEASE_VERSION_NAME string = "20241028" // 发行版的版本号名称
GWAF_RELEASE_VERSION string = "v1.0.0" // 发行版的版本号
GWAF_LAST_UPDATE_TIME time.Time // 上次时间
GWAF_LAST_TIME_UNIX int64 = 0 // 上次时间戳
GWAF_NOTICE_ENABLE bool = false // 是否开启通知
GWAF_CAN_EXPORT_DOWNLOAD_LOG bool = false //是否可以导出下载日志
GWAF_DLP dlpheader.EngineAPI // 脱敏引擎

4
model/response/waf_stat.go
View File

@@ -28,8 +28,8 @@ type WafCityStats struct {
NormalCityOfRange map[string]int64 //区间正常城市数量
}
type WafIPStats struct {
AttackIPOfRange []model.StatsIPCount //区间攻击IP数量
NormalIPOfRange []model.StatsIPCount //区间正常IP数量
AttackIPOfRange []model.StatsIPCountMore //区间攻击IP数量
NormalIPOfRange []model.StatsIPCountMore //区间正常IP数量
}
/*

10
model/stats.go
View File

@@ -73,6 +73,16 @@ type StatsDayCount struct {
域名对应的数量[临时]
*/
type StatsIPCount struct {
IP string `json:"ip"` //ip
IPBelong string `json:"ip_belong"` //归属地
Count int64 `json:"count"` //数量
}
/*
*
域名对应的数量丰富的标签内容
*/
type StatsIPCountMore struct {
IP string `json:"ip"` //ip
IPBelong string `json:"ip_belong"` //归属地
IPTag []IPTag `json:"ip_tags"` //IP标签

8
service/waf_service/waf_log.go
View File

@@ -174,6 +174,14 @@ func (receiver *WafLogService) DeleteHistory(day string) {
global.GWAF_LOCAL_LOG_DB.Where("create_time < ?", day).Delete(&innerbean.WebLog{})
}
// GetUnixTimeByCounter 依据开始时间和到期时间获取一个最新的时间戳
func (receiver *WafLogService) GetUnixTimeByCounter(lastStartCreateUnix int64, lastEndCreateUnix int64) innerbean.WebLog {
var weblog innerbean.WebLog
global.GWAF_LOCAL_LOG_DB.Where("unix_add_time>=? and unix_add_time<?", lastStartCreateUnix, lastEndCreateUnix).Order("unix_add_time desc").Limit(1).Find(&weblog)
return weblog
}
/*
*
判断是否合法

27
service/waf_service/waf_stat.go
View File

@@ -109,14 +109,22 @@ func (receiver *WafStatService) StatHomeSumDayTopIPRangeApi(req request.WafStats
Select("ip,sum(count) as count").Group("ip").Order("sum(count) desc").
Limit(10).
Scan(&AttackCountOfRange)
var AttackCountOfRangeMore []model.StatsIPCountMore
for i := range AttackCountOfRange {
region := utils.GetCountry(AttackCountOfRange[i].IP)
AttackCountOfRange[i].IPBelong = region[0]
//查询IP标签
var ipTags []model.IPTag
global.GWAF_LOCAL_DB.Where("tenant_id = ? and user_code = ? and ip=?",
global.GWAF_TENANT_ID, global.GWAF_USER_CODE, AttackCountOfRange[i].IP).Find(&ipTags)
AttackCountOfRange[i].IPTag = ipTags
statMore := model.StatsIPCountMore{
IP: AttackCountOfRange[i].IP,
IPBelong: region[0],
IPTag: ipTags,
Count: AttackCountOfRange[i].Count,
}
AttackCountOfRangeMore = append(AttackCountOfRangeMore, statMore)
}
var NormalCountOfRange []model.StatsIPCount
@@ -126,6 +134,8 @@ func (receiver *WafStatService) StatHomeSumDayTopIPRangeApi(req request.WafStats
Group("ip").Order("sum(count) desc").
Limit(10).
Scan(&NormalCountOfRange)
var NormalCountOfRangeMore []model.StatsIPCountMore
for i := range NormalCountOfRange {
region := utils.GetCountry(NormalCountOfRange[i].IP)
NormalCountOfRange[i].IPBelong = region[0]
@@ -134,11 +144,18 @@ func (receiver *WafStatService) StatHomeSumDayTopIPRangeApi(req request.WafStats
var ipTags []model.IPTag
global.GWAF_LOCAL_DB.Where("tenant_id = ? and user_code = ? and ip=?",
global.GWAF_TENANT_ID, global.GWAF_USER_CODE, NormalCountOfRange[i].IP).Find(&ipTags)
NormalCountOfRange[i].IPTag = ipTags
statMore := model.StatsIPCountMore{
IP: NormalCountOfRange[i].IP,
IPBelong: region[0],
IPTag: ipTags,
Count: NormalCountOfRange[i].Count,
}
NormalCountOfRangeMore = append(NormalCountOfRangeMore, statMore)
}
return response2.WafIPStats{
AttackIPOfRange: AttackCountOfRange,
NormalIPOfRange: NormalCountOfRange,
AttackIPOfRange: AttackCountOfRangeMore,
NormalIPOfRange: NormalCountOfRangeMore,
},
nil
}

3
waftask/core_scheduler.go
View File

@@ -1,6 +1,7 @@
package waftask
import (
"SamWaf/common/zlog"
"SamWaf/enums"
"fmt"
"github.com/go-co-op/gocron"
@@ -55,7 +56,7 @@ func (ts *TaskScheduler) ScheduleTask(unit string, interval int, at string, task
if err != nil {
return fmt.Errorf("failed to schedule task: %v", err)
}
fmt.Printf("Task scheduled: %v every %d %s\n", job, interval, unit)
zlog.Debug(fmt.Sprintf("Task scheduled: %v every %d %s\n", job, interval, unit))
return nil
}
func (ts *TaskScheduler) Start() {

57
waftask/localtaskcounter.go → waftask/task_counter.go
View File

@@ -8,6 +8,7 @@ import (
"SamWaf/model"
"SamWaf/model/baseorm"
"SamWaf/service/waf_service"
"fmt"
uuid "github.com/satori/go.uuid"
"time"
)
@@ -15,8 +16,12 @@ import (
var (
wafSysLogService = waf_service.WafSysLogServiceApp
wafSystemConfigService = waf_service.WafSystemConfigServiceApp
wafLogService = waf_service.WafLogServiceApp
)
type LastCounter struct {
UNIX_ADD_TIME int64 `json:"unix_add_time" gorm:"index"` //添加日期unix
}
type CountHostResult struct {
UserCode string `json:"user_code"` //用户码（主要键）
TenantId string `json:"tenant_id"` //租户ID（主要键）
@@ -68,12 +73,6 @@ func TaskCounter() {
zlog.Debug("统计还没完成，调度任务PASS")
}
global.GWAF_SWITCH_TASK_COUNTER = true
/*dateTime, err := time.Parse("2006-01-02", "2023-01-01")
if err != nil {
fmt.Println("解析日期出错:", err)
return
}
currenyDayBak := dateTime*/
/**
1.首次是当前日期，查询当前时间以后的所有数据，备份当前日期
@@ -87,14 +86,31 @@ func TaskCounter() {
global.GWAF_SWITCH_TASK_COUNTER = false
return
}
currenyDayBak := time.Now()
currenyDayMillisecondsBak := (global.GWAF_LAST_UPDATE_TIME.Add(-10 * time.Second).UnixNano()) / 1e6 //倒退10秒
if global.GWAF_LAST_TIME_UNIX == 0 {
global.GWAF_LAST_TIME_UNIX = (global.GWAF_LAST_UPDATE_TIME.UnixNano()) / 1e6
global.GWAF_SWITCH_TASK_COUNTER = false
return
}
//取大于上次时间的时
statTimeUnix := global.GWAF_LAST_TIME_UNIX
endTimeUnix := (time.Now().Add(-5 * time.Second).UnixNano()) / 1e6
//打印 statTimeUnix，endTimeUnix
zlog.Debug(fmt.Sprintf("counter statTimeUnix = %v endTimeUnix=%v", statTimeUnix, endTimeUnix))
lastWebLogDbBean := wafLogService.GetUnixTimeByCounter(statTimeUnix, endTimeUnix)
if lastWebLogDbBean.REQ_UUID == "" {
zlog.Debug("当前期间没有符合条件的数据")
global.GWAF_LAST_TIME_UNIX = endTimeUnix
global.GWAF_SWITCH_TASK_COUNTER = false
return
} else {
global.GWAF_LAST_TIME_UNIX = endTimeUnix
}
//一、 主机聚合统计
{
var resultHosts []CountHostResult
global.GWAF_LOCAL_LOG_DB.Raw("SELECT host_code, user_code,tenant_id ,action,count(req_uuid) as count,day,host FROM \"web_logs\" where task_flag = ? and unix_add_time > ? and tenant_id = ? and user_code =? GROUP BY host_code, user_code,action,tenant_id,day,host",
1, currenyDayMillisecondsBak, global.GWAF_TENANT_ID, global.GWAF_USER_CODE).Scan(&resultHosts)
global.GWAF_LOCAL_LOG_DB.Raw("SELECT host_code, user_code,tenant_id ,action,count(req_uuid) as count,day,host FROM \"web_logs\" where task_flag = ? and unix_add_time >= ? and unix_add_time < ? and tenant_id = ? and user_code =? GROUP BY host_code, user_code,action,tenant_id,day,host",
1, statTimeUnix, endTimeUnix, global.GWAF_TENANT_ID, global.GWAF_USER_CODE).Scan(&resultHosts)
/****
1.如果不存在则创建
2.如果存在则累加这个周期的统计数
@@ -123,7 +139,7 @@ func TaskCounter() {
} else {
statDayMap := map[string]interface{}{
"Count": value.Count + statDay.Count,
"UPDATE_TIME": customtype.JsonTime(currenyDayBak),
"UPDATE_TIME": customtype.JsonTime(time.Now()),
}
updateBean := innerbean.UpdateModel{
Model: model.StatsDay{},
@@ -139,8 +155,8 @@ func TaskCounter() {
//二、 IP聚合统计
{
var resultIP []CountIPResult
global.GWAF_LOCAL_LOG_DB.Raw("SELECT host_code, user_code,tenant_id ,action,count(req_uuid) as count,day,host,src_ip as ip FROM \"web_logs\" where task_flag = ? and unix_add_time > ? and tenant_id = ? and user_code =? GROUP BY host_code, user_code,action,tenant_id,day,host,ip",
1, currenyDayMillisecondsBak, global.GWAF_TENANT_ID, global.GWAF_USER_CODE).Scan(&resultIP)
global.GWAF_LOCAL_LOG_DB.Raw("SELECT host_code, user_code,tenant_id ,action,count(req_uuid) as count,day,host,src_ip as ip FROM \"web_logs\" where task_flag = ? and unix_add_time >= ? and unix_add_time < ? and tenant_id = ? and user_code =? GROUP BY host_code, user_code,action,tenant_id,day,host,ip",
1, statTimeUnix, endTimeUnix, global.GWAF_TENANT_ID, global.GWAF_USER_CODE).Scan(&resultIP)
/****
1.如果不存在则创建
2.如果存在则累加这个周期的统计数
@@ -170,7 +186,7 @@ func TaskCounter() {
} else {
statDayMap := map[string]interface{}{
"Count": value.Count + statDay.Count,
"UPDATE_TIME": customtype.JsonTime(currenyDayBak),
"UPDATE_TIME": customtype.JsonTime(time.Now()),
}
updateBean := innerbean.UpdateModel{
@@ -188,8 +204,8 @@ func TaskCounter() {
//三、 城市信息聚合统计
{
var resultCitys []CountCityResult
global.GWAF_LOCAL_LOG_DB.Raw("SELECT host_code, user_code,tenant_id ,action,count(req_uuid) as count,day,host,country,province,city FROM \"web_logs\" where task_flag = ? and unix_add_time > ? and tenant_id = ? and user_code =? GROUP BY host_code, user_code,action,tenant_id,day,host,country,province,city",
1, currenyDayMillisecondsBak, global.GWAF_TENANT_ID, global.GWAF_USER_CODE).Scan(&resultCitys)
global.GWAF_LOCAL_LOG_DB.Raw("SELECT host_code, user_code,tenant_id ,action,count(req_uuid) as count,day,host,country,province,city FROM \"web_logs\" where task_flag = ? and unix_add_time >= ? and unix_add_time < ? and tenant_id = ? and user_code =? GROUP BY host_code, user_code,action,tenant_id,day,host,country,province,city",
1, statTimeUnix, endTimeUnix, global.GWAF_TENANT_ID, global.GWAF_USER_CODE).Scan(&resultCitys)
/****
1.如果不存在则创建
2.如果存在则累加这个周期的统计数
@@ -221,7 +237,7 @@ func TaskCounter() {
} else {
statDayMap := map[string]interface{}{
"Count": value.Count + statDay.Count,
"UPDATE_TIME": customtype.JsonTime(currenyDayBak),
"UPDATE_TIME": customtype.JsonTime(time.Now()),
}
updateBean := innerbean.UpdateModel{
@@ -239,8 +255,8 @@ func TaskCounter() {
//第四 给IP打标签 开始
{
var resultIPRule []CountIPRuleResult
global.GWAF_LOCAL_LOG_DB.Raw("SELECT src_ip as ip ,rule,count(src_ip) as cnt FROM \"web_logs\" where task_flag = ? and unix_add_time > ? and tenant_id = ? and user_code =? GROUP BY user_code,tenant_id, rule,src_ip",
1, currenyDayMillisecondsBak, global.GWAF_TENANT_ID, global.GWAF_USER_CODE).Scan(&resultIPRule)
global.GWAF_LOCAL_LOG_DB.Raw("SELECT src_ip as ip ,rule,count(src_ip) as cnt FROM \"web_logs\" where task_flag = ? and unix_add_time >= ? and unix_add_time < ? and tenant_id = ? and user_code =? GROUP BY user_code,tenant_id, rule,src_ip",
1, statTimeUnix, endTimeUnix, global.GWAF_TENANT_ID, global.GWAF_USER_CODE).Scan(&resultIPRule)
/****
1.如果不存在则创建
2.如果存在则累加这个IP这个rule的统计数
@@ -270,7 +286,7 @@ func TaskCounter() {
} else {
ipTagUpdateMap := map[string]interface{}{
"Cnt": value.Cnt + ipTag.Cnt,
"UPDATE_TIME": customtype.JsonTime(currenyDayBak),
"UPDATE_TIME": customtype.JsonTime(time.Now()),
}
updateBean := innerbean.UpdateModel{
Model: model.IPTag{},
@@ -283,6 +299,5 @@ func TaskCounter() {
}
} //给IP打标签结束
global.GWAF_LAST_UPDATE_TIME = currenyDayBak
global.GWAF_SWITCH_TASK_COUNTER = false
}