Create & Init Project...

app/service/main/antispam/util/BUILD

@@ -0,0 +1,51 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_test",
+    "go_library",
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = [
+        "deviations_test.go",
+        "pagination_test.go",
+        "string_test.go",
+    ],
+    embed = [":go_default_library"],
+    rundir = ".",
+    tags = ["automanaged"],
+    deps = ["//vendor/github.com/stretchr/testify/assert:go_default_library"],
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "deviations.go",
+        "pagination.go",
+        "rand.go",
+        "string.go",
+        "time.go",
+    ],
+    importpath = "go-common/app/service/main/antispam/util",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [
+        ":package-srcs",
+        "//app/service/main/antispam/util/trie:all-srcs",
+    ],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/service/main/antispam/util/deviations.go

@@ -0,0 +1,64 @@
+package util
+
+import "math"
+
+// Max .
+func Max(vars []int64) (maxVar int64) {
+	for _, i := range vars {
+		if i > maxVar {
+			maxVar = i
+		}
+	}
+	return
+}
+
+// Expectation .
+func Expectation(randomVars []float64) float64 {
+	if len(randomVars) == 0 {
+		return 0
+	}
+
+	var sum float64
+	for _, rv := range randomVars {
+		sum += rv
+	}
+
+	return sum / float64(len(randomVars))
+}
+
+// StdDeviation .
+func StdDeviation(randomVars []float64) float64 {
+	if len(randomVars) == 0 {
+		return 0
+	}
+	return math.Sqrt(Deviation(randomVars))
+}
+
+// Deviation .
+func Deviation(randomVars []float64) float64 {
+	if len(randomVars) == 0 {
+		return 0
+	}
+	var total float64
+	expec := Expectation(randomVars)
+	for _, rv := range randomVars {
+		total += math.Pow(rv-expec, 2.0)
+	}
+	return total / float64(len(randomVars))
+}
+
+// Normallization .
+func Normallization(randomVars []int64) []float64 {
+	if len(randomVars) == 0 {
+		return nil
+	}
+	maxVal := Max(randomVars)
+	if maxVal == 0 || maxVal == 1 {
+		return nil
+	}
+	res := make([]float64, 0, len(randomVars))
+	for _, rv := range randomVars {
+		res = append(res, math.Log10(float64(rv))/math.Log10(float64(maxVal)))
+	}
+	return res
+}

app/service/main/antispam/util/deviations_test.go

@@ -0,0 +1,255 @@
+package util
+
+import "testing"
+
+func TestMax(t *testing.T) {
+	cases := []struct {
+		input []int64
+	}{
+		{
+			input: []int64{},
+		},
+		{
+			input: []int64{
+				268826797,
+			},
+		},
+		{
+			input: []int64{
+				1,
+				2,
+				3,
+				4,
+				5,
+				272668542,
+				87759075,
+				272670403,
+				192148035,
+				189361023,
+				88269430,
+				89400131,
+				272690653,
+				269278391,
+				268823477,
+				268826797,
+			},
+		},
+	}
+	for _, c := range cases {
+		t.Run("", func(t *testing.T) {
+			rs := Max(c.input)
+			t.Logf("input %v \noutputs:%v\n", c.input, rs)
+		})
+	}
+}
+
+func TestStdDeviation(t *testing.T) {
+	cases := []struct {
+		input []float64
+	}{
+		{
+			input: []float64{},
+		},
+		{
+			input: []float64{
+				268826797,
+			},
+		},
+		{
+			input: []float64{
+				1,
+				2,
+				3,
+				4,
+				5,
+				272668542,
+				87759075,
+				272670403,
+				192148035,
+				189361023,
+				88269430,
+				89400131,
+				272690653,
+				269278391,
+				268823477,
+				268826797,
+			},
+		},
+	}
+	for _, c := range cases {
+		t.Run("", func(t *testing.T) {
+			rs := StdDeviation(c.input)
+			t.Logf("input %v \noutputs:%v\n", c.input, rs)
+		})
+	}
+}
+
+func TestDeviation(t *testing.T) {
+	cases := []struct {
+		input []float64
+	}{
+		{
+			input: []float64{
+				1,
+				2,
+				3,
+				4,
+				5,
+				272668542,
+				87759075,
+				272670403,
+				192148035,
+				189361023,
+				88269430,
+				89400131,
+				272690653,
+				269278391,
+				268823477,
+				268826797,
+			},
+		},
+	}
+	for _, c := range cases {
+		t.Run("", func(t *testing.T) {
+			rs := Deviation(c.input)
+			t.Logf("input %v \noutputs:%v\n", c.input, rs)
+		})
+	}
+}
+
+func TestNormallization(t *testing.T) {
+	cases := []struct {
+		input []int64
+	}{
+		{
+			input: []int64{
+				1,
+				2,
+				3,
+				4,
+				5,
+				272668542,
+				87759075,
+				272670403,
+				192148035,
+				189361023,
+				88269430,
+				89400131,
+				272690653,
+				269278391,
+				268823477,
+				268826797,
+			},
+		},
+	}
+	for _, c := range cases {
+		t.Run("", func(t *testing.T) {
+			rs := Normallization(c.input)
+			t.Logf("input %v\n, outputs:%v\n", c.input, rs)
+		})
+	}
+}
+
+func TestExpectation(t *testing.T) {
+	cases := []struct {
+		input []float64
+	}{
+		{
+			input: []float64{
+				272668542,
+				87759075,
+				272670403,
+				192148035,
+				189361023,
+				88269430,
+				89400131,
+				272690653,
+				269278391,
+				268823477,
+				268826797,
+				107338074,
+				272635619,
+				272655050,
+				272280850,
+				274403561,
+				274404369,
+				274408851,
+				33561986,
+				274440108,
+				274440766,
+				274440926,
+				273730691,
+				274443040,
+				273744469,
+				274443230,
+				273749519,
+				274837710,
+				273748847,
+				274837920,
+				273814911,
+				274838433,
+				269278030,
+				273815488,
+				273819536,
+				274838710,
+				274838821,
+				274875985,
+				274876083,
+			},
+		},
+		{
+			input: []float64{
+				23062012,
+				32199188,
+				114238752,
+				35134612,
+				18289010,
+				29669239,
+				25702393,
+				19201815,
+				38191035,
+				24237961,
+				10155123,
+				36970516,
+				14015382,
+				89085165,
+				28324458,
+				38405237,
+				20190219,
+				175006499,
+				178984876,
+				34990873,
+				158026517,
+				111053972,
+				25519948,
+				39061494,
+				155286833,
+				87469728,
+				62849283,
+				210174070,
+				13841839,
+				65905090,
+				8219588,
+				37192235,
+				274356431,
+				44363565,
+				22899631,
+				43582749,
+				101217680,
+				25011431,
+				33447081,
+				222278335,
+				20785287,
+				38448378,
+				40153047,
+			},
+		},
+	}
+
+	for _, c := range cases {
+		t.Run("", func(t *testing.T) {
+			output := Expectation(c.input)
+			t.Logf("input %v, output:%v", c.input, output)
+		})
+	}
+}

app/service/main/antispam/util/pagination.go

@@ -0,0 +1,64 @@
+package util
+
+const (
+	// DefaultPerPage .
+	DefaultPerPage = 20
+)
+
+// SimplePage calculate "from", "to" without total_counts
+// "from" index start from 1
+func (p *Pagination) SimplePage() (from int64, to int64) {
+	if p.CurPage == 0 || p.PerPage == 0 {
+		p.CurPage, p.PerPage = 1, DefaultPerPage
+	}
+	from = (p.CurPage-1)*p.PerPage + 1
+	to = from + p.PerPage - 1
+	return
+}
+
+// Page calculate "from", "to" with total_counts
+// index start from 1
+func (p *Pagination) Page(total int64) (from int64, to int64) {
+	if p.CurPage == 0 {
+		p.CurPage = 1
+	}
+	if p.PerPage == 0 {
+		p.PerPage = DefaultPerPage
+	}
+
+	if total == 0 || total < p.PerPage*(p.CurPage-1) {
+		return
+	}
+	if total <= p.PerPage {
+		return 1, total
+	}
+	from = (p.CurPage-1)*p.PerPage + 1
+	if (total - from + 1) < p.PerPage {
+		return from, total
+	}
+	return from, from + p.PerPage - 1
+}
+
+// VagueOffsetLimit calculate "offset", "limit" without total_counts
+func (p *Pagination) VagueOffsetLimit() (offset int64, limit int64) {
+	from, to := p.SimplePage()
+	if to == 0 || from == 0 {
+		return 0, 0
+	}
+	return from - 1, to - from + 1
+}
+
+// OffsetLimit calculate "offset" and "start" with total_counts
+func (p *Pagination) OffsetLimit(total int64) (offset int64, limit int64) {
+	from, to := p.Page(total)
+	if to == 0 || from == 0 {
+		return 0, 0
+	}
+	return from - 1, to - from + 1
+}
+
+// Pagination perform page algorithm
+type Pagination struct {
+	CurPage int64
+	PerPage int64
+}

app/service/main/antispam/util/pagination_test.go

@@ -0,0 +1,118 @@
+package util
+
+import (
+	"fmt"
+	"math/rand"
+	"testing"
+	"time"
+)
+
+func TestSimplePage(t *testing.T) {
+	cases := []struct {
+		perPage      int64
+		curPage      int64
+		expectedFrom int64
+		expectedTo   int64
+	}{
+		{20, 9, 161, 180},
+		{0, 1, 1, 20},
+		{0, 0, 1, 20},
+		{1, 0, 1, 20},
+	}
+	for _, c := range cases {
+		t.Run(fmt.Sprintf("curPage(%d) perPage(%d)", c.curPage, c.perPage), func(t *testing.T) {
+			p := &Pagination{
+				CurPage: c.curPage,
+				PerPage: c.perPage,
+			}
+			from, to := p.SimplePage()
+			if from != c.expectedFrom || to != c.expectedTo {
+				t.Errorf("cond.SimplePage() = from: %d, to: %d, want: %d, %d", from, to, c.expectedFrom, c.expectedTo)
+			}
+		})
+	}
+}
+
+func TestPage(t *testing.T) {
+	cases := []struct {
+		total        int64
+		perPage      int64
+		curPage      int64
+		expectedFrom int64
+		expectedTo   int64
+	}{
+		{66269, 20, 3314, 66261, 66269},
+		{66269, 20, 3313, 66241, 66260},
+		{81, 20, 9, 0, 0},
+		{100, 0, 1, 1, 20},
+		{1, 20, 1, 1, 1},
+		{0, 20, 1, 0, 0},
+		{5, 20, 1, 1, 5},
+		{211, 20, 3, 41, 60},
+		{100, 100, 1, 1, 100},
+		{101, 20, 6, 101, 101},
+		{211, 20, 2, 21, 40},
+		{211, 20, 1, 1, 20},
+	}
+	for _, c := range cases {
+		t.Run(fmt.Sprintf("total(%d) curPage(%d) perPage(%d)", c.total, c.curPage, c.perPage), func(t *testing.T) {
+			p := &Pagination{
+				CurPage: c.curPage,
+				PerPage: c.perPage,
+			}
+			from, to := p.Page(c.total)
+			if from != c.expectedFrom || to != c.expectedTo {
+				t.Errorf("cond.Page(%d) = from: %d, to: %d, want: %d, %d", c.total, from, to, c.expectedFrom, c.expectedTo)
+			}
+		})
+	}
+}
+
+func TestOffsetLimit(t *testing.T) {
+	cases := []struct {
+		total          int64
+		perpage        int64
+		curpage        int64
+		expectedoffset int64
+		expectedlimit  int64
+	}{
+		{66269, 20, 3314, 66260, 9},
+		{66269, 20, 3313, 66240, 20},
+		{100, 0, 1, 0, 20},
+		{1, 20, 1, 0, 1},
+		{0, 20, 1, 0, 0},
+		{5, 20, 1, 0, 5},
+		{211, 20, 3, 40, 20},
+		{100, 100, 1, 0, 100},
+		{101, 20, 6, 100, 1},
+		{211, 20, 2, 20, 20},
+		{211, 20, 1, 0, 20},
+	}
+	for _, c := range cases {
+		t.Run(fmt.Sprintf("total(%d) curpage(%d) perpage(%d)", c.total, c.curpage, c.perpage), func(t *testing.T) {
+			p := &Pagination{
+				CurPage: c.curpage,
+				PerPage: c.perpage,
+			}
+			offset, limit := p.OffsetLimit(c.total)
+			if offset != c.expectedoffset || limit != c.expectedlimit {
+				t.Errorf("cond.offsetlimit(%d) = offset: %d, limit: %d, want %d, %d", c.total, offset, limit, c.expectedoffset, c.expectedlimit)
+			}
+		})
+	}
+}
+
+func TestBulkPage(t *testing.T) {
+	p := &Pagination{}
+	rand.Seed(time.Now().Unix())
+	for i := 0; i < 9999; i++ {
+		p.CurPage = rand.Int63n(10000)
+		p.PerPage = rand.Int63n(300)
+		total := rand.Int63n(20000)
+		from, to := p.Page(total)
+		if from < 0 || to < 0 {
+			t.Fatalf(`Bulk test page fail, got negative result, 
+				total: %d, curPage: %d, perPage: %d, from: %d, to: %d`, total, p.CurPage, p.PerPage, from, to)
+		}
+	}
+}

app/service/main/antispam/util/rand.go

@@ -0,0 +1,19 @@
+package util
+
+import "math/rand"
+
+var letterRunes = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ")
+
+// RandID .
+func RandID() int64 {
+	return rand.Int63()
+}
+
+// RandStr .
+func RandStr(size int) string {
+	b := make([]rune, size)
+	for i := range b {
+		b[i] = letterRunes[rand.Intn(len(letterRunes))]
+	}
+	return string(b)
+}

app/service/main/antispam/util/string.go

@@ -0,0 +1,65 @@
+package util
+
+import (
+	"strconv"
+	"strings"
+)
+
+// StrSliToSQLVarchars convert string slice to varchar in sql syntax
+// eg: ["default", "deleted", "modified"] -> " 'default', 'deleted', 'modified' "
+// so that we can used it in 'SELECT * ... WHERE xxx IN ('default', 'deleted', 'modified')'
+func StrSliToSQLVarchars(s []string) string {
+	sli := make([]string, len(s))
+	for i, ss := range s {
+		sli[i] = "'" + ss + "'"
+	}
+	return strings.Join(sli, ",")
+}
+
+// StrToIntSli convert string to int slice, eg: "1,2,3" -> [1,2,3]
+func StrToIntSli(s string, delimiter string) ([]int64, error) {
+	var result []int64
+	sli := strings.Split(s, delimiter)
+	for _, intStr := range sli {
+		i, err := strconv.ParseInt(intStr, 10, 64)
+		if err != nil {
+			return nil, err
+		}
+		result = append(result, i)
+	}
+	return result, nil
+}
+
+// IntSliToSQLVarchars convert int slice to string, eg: [1,2,3] -> "1,2,3"
+func IntSliToSQLVarchars(ints []int64) string {
+	return intSliToStr(ints, ",")
+}
+
+func intSliToStr(ints []int64, delimiter string) string {
+	sli := make([]string, len(ints))
+	for i, ii := range ints {
+		sli[i] = strconv.FormatInt(ii, 10)
+	}
+	return strings.Join(sli, delimiter)
+}
+
+// SameChar check if string consists of same characters
+func SameChar(content string) bool {
+	content = strings.ToLower(content)
+	first := content[0]
+	for _, s := range content {
+		if s != rune(first) {
+			return false
+		}
+	}
+	return true
+}
+
+// StripPrefix remove prefix from string if exists
+func StripPrefix(s string, prefix string, suffix string) string {
+	if strings.HasPrefix(s, prefix) {
+		i := strings.Index(s, suffix)
+		return s[i+1:]
+	}
+	return s
+}

app/service/main/antispam/util/string_test.go

@@ -0,0 +1,135 @@
+package util
+
+import (
+	"fmt"
+	"reflect"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func ExampleIntSliToSQLVarchars() {
+	fmt.Println(IntSliToSQLVarchars([]int64{1, 2, 3}))
+	// Output: 1,2,3
+}
+
+func ExampleStrToIntSli() {
+	fmt.Println(StrToIntSli("1,2,3", ","))
+	// Output: [1 2 3] <nil>
+}
+
+func ExampleStrSliToSQLVarchars() {
+	fmt.Println(StrSliToSQLVarchars([]string{"default", "deleted", "modified"}))
+	// Output: 'default','deleted','modified'
+}
+
+func TestStrSliToSQLVarchars(t *testing.T) {
+	cases := []struct {
+		s        []string
+		expected string
+	}{
+		{[]string{"foo", "bar"}, "'foo','bar'"},
+	}
+	for _, c := range cases {
+		t.Run(fmt.Sprintf("inputStr(%v)", c.s), func(t *testing.T) {
+			got := StrSliToSQLVarchars(c.s)
+			if got != c.expected {
+				t.Errorf("StrSliToSQLVarchars(%v) = %s, want: %s", c.s, got, c.expected)
+			}
+		})
+	}
+}
+
+func TestStrToIntSli(t *testing.T) {
+	cases := []struct {
+		s           string
+		delimiter   string
+		expectedSli []int64
+		expectedErr error
+	}{
+		{"1,2,3", ",", []int64{1, 2, 3}, nil},
+		{"1 2 3", " ", []int64{1, 2, 3}, nil},
+		{"1|2|3", "|", []int64{1, 2, 3}, nil},
+	}
+	for _, c := range cases {
+		assert := assert.New(t)
+		t.Run(fmt.Sprintf("inputString(%v) delimiter(%s)", c.s, c.delimiter), func(t *testing.T) {
+			got, err := StrToIntSli(c.s, c.delimiter)
+			assert.Equal(c.expectedSli, got, "")
+			assert.Equal(c.expectedErr, err, "")
+		})
+	}
+}
+
+func TestIntSliToStr(t *testing.T) {
+	cases := []struct {
+		s         []int64
+		delimiter string
+		expected  string
+	}{
+		{[]int64{1, 2, 3}, ",", "1,2,3"},
+		{[]int64{1, 2, 3}, " ", "1 2 3"},
+		{[]int64{1, 2, 3}, "|", "1|2|3"},
+	}
+	for _, c := range cases {
+		t.Run(fmt.Sprintf("inputSli(%v) delimiter(%s)", c.s, c.delimiter), func(t *testing.T) {
+			got := intSliToStr(c.s, c.delimiter)
+			if !reflect.DeepEqual(got, c.expected) {
+				t.Errorf("IntSliToStr(%v, %s) = %s, want %s", c.s, c.delimiter, got, c.expected)
+			}
+		})
+	}
+}
+
+func TestStripPrefix(t *testing.T) {
+	cases := []struct {
+		name           string
+		content        string
+		expectedOutput string
+	}{
+		{
+			"need strip prefix",
+			"回复 @画鸾凰 :我知道 但我的不是正版的 上不了工坊 只能要模型软件 格式是LPK的模型软件 你找一下看看 模型列表下面应该有在哪个文件夹里面 找到可以发我QQ1918882322 如果找不到就算吧",
+			"我知道 但我的不是正版的 上不了工坊 只能要模型软件 格式是LPK的模型软件 你找一下看看 模型列表下面应该有在哪个文件夹里面 找到可以发我QQ1918882322 如果找不到就算吧",
+		},
+		{
+			"empty reply body",
+			"回复 @画鸾凰 :",
+			"",
+		},
+		{
+			"not need strip",
+			"我知道 但我的不是正版的 上不了工坊 只能要模型软件 格式是LPK的模型软件 你找一下看看 模型列表下面应该有在哪个文件夹里面 找到可以发我QQ1918882322 如果找不到就算吧",
+			"我知道 但我的不是正版的 上不了工坊 只能要模型软件 格式是LPK的模型软件 你找一下看看 模型列表下面应该有在哪个文件夹里面 找到可以发我QQ1918882322 如果找不到就算吧",
+		},
+	}
+
+	for _, c := range cases {
+		t.Run(c.name, func(t *testing.T) {
+			actual := StripPrefix(c.content, "回复 @", ":")
+			if actual != c.expectedOutput {
+				t.Fatalf("Strip Prefix failed, expected %q \t\n got %q", c.expectedOutput, actual)
+			}
+		})
+	}
+}
+
+func TestSameChar(t *testing.T) {
+	cases := []struct {
+		content        string
+		expectedResult bool
+	}{
+		{"~~~~~~~", true},
+		{"666666666", true},
+		{"666666666~~~", false},
+		{"WWWWWWW", true},
+		{"XXXxxx", true},
+	}
+	for _, c := range cases {
+		t.Run(fmt.Sprintf("content(%s)", c.content), func(t *testing.T) {
+			if rs := SameChar(c.content); rs != c.expectedResult {
+				t.Errorf("SameChar(%s) = %v, want %v", c.content, rs, c.expectedResult)
+			}
+		})
+	}
+}

app/service/main/antispam/util/time.go

@@ -0,0 +1,25 @@
+package util
+
+import (
+	"fmt"
+	"time"
+)
+
+const (
+	// TimeFormat .
+	TimeFormat = "2006-01-02 15:04:05"
+)
+
+// JSONTime .
+type JSONTime time.Time
+
+// MarshalJSON .
+func (jt JSONTime) MarshalJSON() ([]byte, error) {
+	stamp := fmt.Sprintf("%q", time.Time(jt).Format(TimeFormat))
+	return []byte(stamp), nil
+}
+
+// Before .
+func (jt JSONTime) Before(t time.Time) bool {
+	return time.Time(jt).Before(t)
+}

app/service/main/antispam/util/trie/BUILD

@@ -0,0 +1,41 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_test",
+    "go_library",
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = ["rune_trie_test.go"],
+    embed = [":go_default_library"],
+    rundir = ".",
+    tags = ["automanaged"],
+    deps = ["//app/service/main/antispam/util:go_default_library"],
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "rune_trie.go",
+        "trie.go",
+    ],
+    importpath = "go-common/app/service/main/antispam/util/trie",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/service/main/antispam/util/trie/rune_trie.go

@@ -0,0 +1,120 @@
+package trie
+
+func (trie *RuneTrie) find(contents []rune, accu []rune) (string, interface{}) {
+	if trie.value != nil {
+		return string(accu), trie.value
+	}
+	if len(contents) == 0 {
+		return "", nil
+	}
+	k := contents[0]
+	tt, ok := trie.children[k]
+	if !ok {
+		return "", nil
+	}
+	return tt.find(contents[1:], append(accu, k))
+}
+
+func (trie *RuneTrie) Find(content string, accu string) (string, interface{}) {
+	rcontent := []rune(content)
+	if len(rcontent) == 0 {
+		return "", nil
+	}
+	k, v := trie.find(rcontent, []rune(accu))
+	if v == nil {
+		return trie.Find(string(rcontent[1:]), accu)
+	}
+	return k, v
+}
+
+/*
+* the belowing codes come from "https://github.com/dghubble/trie"
+ */
+type RuneTrie struct {
+	value    interface{}
+	children map[rune]*RuneTrie
+}
+
+func NewRuneTrie() *RuneTrie {
+	return &RuneTrie{
+		children: make(map[rune]*RuneTrie),
+	}
+}
+
+func (trie *RuneTrie) Get(key string) interface{} {
+	node := trie
+	for _, r := range key {
+		node = node.children[r]
+		if node == nil {
+			return nil
+		}
+	}
+	return node.value
+}
+
+func (trie *RuneTrie) Put(key string, value interface{}) bool {
+	node := trie
+	for _, r := range key {
+		child := node.children[r]
+		if child == nil {
+			child = NewRuneTrie()
+			node.children[r] = child
+		}
+		node = child
+	}
+	isNewVal := node.value == nil
+	node.value = value
+	return isNewVal
+}
+
+func (trie *RuneTrie) Delete(key string) bool {
+	path := make([]nodeRune, len(key))
+	node := trie
+	for i, r := range key {
+		path[i] = nodeRune{r: r, node: node}
+		node = node.children[r]
+		if node == nil {
+			return false
+		}
+	}
+	node.value = nil
+	if node.isLeaf() {
+		for i := len(key) - 1; i >= 0; i-- {
+			parent := path[i].node
+			r := path[i].r
+			delete(parent.children, r)
+			if parent.value != nil || !parent.isLeaf() {
+				break
+			}
+		}
+	}
+	return true
+}
+
+func (trie *RuneTrie) Walk(walker WalkFunc) error {
+	return trie.walk("", walker)
+}
+
+type nodeRune struct {
+	node *RuneTrie
+	r    rune
+}
+
+func (trie *RuneTrie) walk(key string, walker WalkFunc) error {
+	if trie.value != nil {
+		walker(key, trie.value)
+	}
+	for r, child := range trie.children {
+		err := child.walk(key+string(r), walker)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (trie *RuneTrie) isLeaf() bool {
+	return len(trie.children) == 0
+}
+
+type WalkFunc func(key string, value interface{}) error

app/service/main/antispam/util/trie/rune_trie_test.go

@@ -0,0 +1,108 @@
+package trie
+
+import (
+	"errors"
+	"math/rand"
+	"testing"
+	"time"
+
+	"go-common/app/service/main/antispam/util"
+)
+
+func TestRuneTrieAdd(t *testing.T) {
+	tr := NewRuneTrie()
+	tr.Put("jimmy", 1)
+	tr.Put("anny", 87)
+	tr.Put("xxxx", 23)
+	tr.Put("jim", 2)
+
+	if v := tr.Get("jimxx"); v != nil {
+		t.Errorf("expected nil, got %v", v)
+	}
+	if v := tr.Get("jimmy"); v.(int) != 1 {
+		t.Errorf("expected val, got %v", v)
+	}
+	if v := tr.Get("anny"); v.(int) != 87 {
+		t.Errorf("expected val, got %v", v)
+	}
+	if v := tr.Get("xxxx"); v.(int) != 23 {
+		t.Errorf("expected val, got %v", v)
+	}
+	if v := tr.Get("jim"); v.(int) != 2 {
+		t.Errorf("expected val, got %v", v)
+	}
+}
+
+func BenchmarkRuneTriePut(b *testing.B) {
+	rand.Seed(time.Now().UnixNano())
+	tr := NewRuneTrie()
+	for i := 0; i < b.N; i++ {
+		tr.Put(util.RandStr(10), 845)
+	}
+}
+
+func TestRuneTrieFind(t *testing.T) {
+	tr := NewRuneTrie()
+	tr.Put("我才是大佬", 2)
+	tr.Put("我才是大佬", 88)
+	tr.Put("mm", 1)
+	tr.Put("mmp", 2)
+	tr.Put("my name is jimmymmp", 2)
+	tr.Put("xxx", 88)
+	tr.Put("jimmy xxx, hhjhmmp", 2)
+
+	cases := []struct {
+		content     string
+		expectKey   string
+		expectValue int
+	}{
+		{
+			content:     "mm",
+			expectKey:   "mm",
+			expectValue: 1,
+		},
+		{
+			content:     "m都xx发生地方范德萨发爱迪生刚发的否多少发生的否阿萨德否收到符文大师否xxxmy name is jimy, hhjhmp",
+			expectKey:   "xxx",
+			expectValue: 88,
+		},
+		{
+			content:     "m都mxx发生地方范德萨发爱迪生刚发的否多少发生的否阿萨德否收到符文大师否xxxmy name is jimy, hhjhmp",
+			expectKey:   "xxx",
+			expectValue: 88,
+		},
+		{
+			content:     "我才是大佬",
+			expectKey:   "我才是大佬",
+			expectValue: 88,
+		},
+	}
+
+	for _, c := range cases {
+		t.Run("", func(t *testing.T) {
+			k, v := tr.Find(c.content, "")
+			if v == nil {
+				t.Fatal(errors.New("val is nil"))
+			}
+
+			if k != c.expectKey || v.(int) != c.expectValue {
+				t.Errorf("want key: %s, val:%v, got key:%s, val:%v", c.expectKey, c.expectValue, k, v)
+			}
+		})
+	}
+}
+
+//BenchmarkTrieListFind-4		   50000     25347 ns/op
+func BenchmarkRuneTrieFind(b *testing.B) {
+	rand.Seed(time.Now().UnixNano())
+	tr := NewRuneTrie()
+	for i := 0; i < b.N; i++ {
+		tr.Put(util.RandStr(10), i)
+	}
+	tr.Put("地方考虑saDFFDSALK", 8888)
+	tr.Put("都说了开发贷款", 7512)
+
+	for i := 0; i < b.N; i++ {
+		tr.Find("dfa啥都发生地方的施工费按发的噶是打发士大夫撒旦噶尔尕热狗怕的是结果来看；砥节奉公；来人速度感而过;sfdsfas fsadf asd fsad f asd都说了sdfs gfdgd jimmy开发速度来发噶都说了开发贷款时间范德萨了空间发的是 jimmy按时到路口发生撒地方考虑saDFFDSALKFDFASDFASDFSDFSADFRGEWTRETGERG", "")
+	}
+}

app/service/main/antispam/util/trie/trie.go

@@ -0,0 +1,11 @@
+package trie
+
+type Trier interface {
+	Put(key string, value interface{}) bool
+	Find(in string, acc string) (key string, value interface{})
+	Get(key string) interface{}
+}
+
+func New() Trier {
+	return NewRuneTrie()
+}